{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Waler2DBulletEnv with Twin Delayed DDPG (TD3)\n",
    "\n",
    "### Paper: https://arxiv.org/abs/1802.09477\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Start the Environment and Agent "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_dir=D:\\Users\\user\\Anaconda2_02Aug2019\\envs\\ml2\\lib\\site-packages\\pybullet_envs\\bullet\n",
      "device:  cuda\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import pybullet_envs\n",
    "import pybullet_envs.bullet as bul\n",
    "\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import gym\n",
    "import os\n",
    "\n",
    "import time\n",
    "from TwinDelayed import Actor, Critic, ReplayBuffer, TD3\n",
    "from collections import deque\n",
    "import itertools as it\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_dim:  22 , action_dim:  6\n",
      "max_action:  1.0 , threshold:  2500.0 , std_noise:  0.02\n"
     ]
    }
   ],
   "source": [
    "start_timestep=1e4\n",
    "\n",
    "std_noise=0.02\n",
    "\n",
    "env = gym.make('Walker2DBulletEnv-v0')\n",
    "\n",
    "# Set seeds\n",
    "seed = 12345\n",
    "env.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "state = env.reset()\n",
    "state_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.shape[0] \n",
    "max_action = float(env.action_space.high[0])\n",
    "threshold = env.spec.reward_threshold\n",
    "\n",
    "print('start_dim: ', state_dim, ', action_dim: ', action_dim)\n",
    "print('max_action: ', max_action, ', threshold: ', threshold, ', std_noise: ', std_noise)\n",
    "\n",
    "agent = TD3(state_dim, action_dim, max_action)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Traning the agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save(agent = agent, filename='checkpnt, directory = 'dir_chkpoint')     \n",
    "def save(agent, filename, directory):\n",
    "    torch.save(agent.actor.state_dict(), '%s/%s_actor.pth' % (directory, filename))\n",
    "    torch.save(agent.critic.state_dict(), '%s/%s_critic.pth' % (directory, filename))\n",
    "    torch.save(agent.actor_target.state_dict(), '%s/%s_actor_t.pth' % (directory, filename))\n",
    "    torch.save(agent.critic_target.state_dict(), '%s/%s_critic_t.pth' % (directory, filename))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Low in action space:  [-1. -1. -1. -1. -1. -1.] , High:  [1. 1. 1. 1. 1. 1.] , Action_dim:  6\n",
      "Ep. 10, Timestep 130,  Ep.Timesteps 9, Score: 15.12, Avg.Score: 15.75, Max.Score: 22.04, Time: 00:00:10 \n",
      "Ep. 20, Timestep 267,  Ep.Timesteps 21, Score: 20.10, Avg.Score: 16.19, Max.Score: 23.60, Time: 00:00:14 \n",
      "Ep. 30, Timestep 400,  Ep.Timesteps 7, Score: 11.34, Avg.Score: 16.01, Max.Score: 23.60, Time: 00:00:20 \n",
      "Ep. 40, Timestep 532,  Ep.Timesteps 14, Score: 16.95, Avg.Score: 16.11, Max.Score: 23.60, Time: 00:00:24 \n",
      "Ep. 50, Timestep 686,  Ep.Timesteps 12, Score: 16.28, Avg.Score: 16.43, Max.Score: 27.63, Time: 00:00:29 \n",
      "Ep. 60, Timestep 828,  Ep.Timesteps 10, Score: 15.38, Avg.Score: 16.57, Max.Score: 27.63, Time: 00:00:34 \n",
      "Ep. 70, Timestep 978,  Ep.Timesteps 17, Score: 17.69, Avg.Score: 16.66, Max.Score: 28.73, Time: 00:00:40 \n",
      "Ep. 80, Timestep 1125,  Ep.Timesteps 28, Score: 28.02, Avg.Score: 16.72, Max.Score: 28.73, Time: 00:00:45 \n",
      "Ep. 90, Timestep 1273,  Ep.Timesteps 11, Score: 13.91, Avg.Score: 16.70, Max.Score: 28.73, Time: 00:00:51 \n",
      "Ep. 100, Timestep 1399,  Ep.Timesteps 8, Score: 12.31, Avg.Score: 16.66, Max.Score: 28.73, Time: 00:00:56 \n",
      "Ep. 110, Timestep 1532,  Ep.Timesteps 8, Score: 13.51, Avg.Score: 16.63, Max.Score: 28.73, Time: 00:01:00 \n",
      "Ep. 120, Timestep 1707,  Ep.Timesteps 11, Score: 16.59, Avg.Score: 16.81, Max.Score: 29.25, Time: 00:01:06 \n",
      "Ep. 130, Timestep 1858,  Ep.Timesteps 21, Score: 19.10, Avg.Score: 16.97, Max.Score: 29.25, Time: 00:01:12 \n",
      "Ep. 140, Timestep 1984,  Ep.Timesteps 11, Score: 13.82, Avg.Score: 16.87, Max.Score: 29.25, Time: 00:01:16 \n",
      "Ep. 150, Timestep 2128,  Ep.Timesteps 7, Score: 12.60, Avg.Score: 16.84, Max.Score: 29.25, Time: 00:01:21 \n",
      "Ep. 160, Timestep 2265,  Ep.Timesteps 10, Score: 13.11, Avg.Score: 16.73, Max.Score: 29.25, Time: 00:01:26 \n",
      "Ep. 170, Timestep 2386,  Ep.Timesteps 16, Score: 16.56, Avg.Score: 16.49, Max.Score: 29.25, Time: 00:01:30 \n",
      "Ep. 180, Timestep 2526,  Ep.Timesteps 14, Score: 17.12, Avg.Score: 16.44, Max.Score: 29.25, Time: 00:01:35 \n",
      "Ep. 190, Timestep 2723,  Ep.Timesteps 17, Score: 18.05, Avg.Score: 16.88, Max.Score: 33.49, Time: 00:01:42 \n",
      "Ep. 200, Timestep 2854,  Ep.Timesteps 9, Score: 13.97, Avg.Score: 16.81, Max.Score: 33.49, Time: 00:01:48 \n",
      "Ep. 210, Timestep 2988,  Ep.Timesteps 11, Score: 16.75, Avg.Score: 16.97, Max.Score: 33.49, Time: 00:01:52 \n",
      "Ep. 220, Timestep 3148,  Ep.Timesteps 20, Score: 20.35, Avg.Score: 16.88, Max.Score: 33.49, Time: 00:01:58 \n",
      "Ep. 230, Timestep 3293,  Ep.Timesteps 19, Score: 19.07, Avg.Score: 16.78, Max.Score: 33.49, Time: 00:02:03 \n",
      "Ep. 240, Timestep 3457,  Ep.Timesteps 28, Score: 22.79, Avg.Score: 17.01, Max.Score: 33.49, Time: 00:02:09 \n",
      "Ep. 250, Timestep 3588,  Ep.Timesteps 11, Score: 15.94, Avg.Score: 16.88, Max.Score: 33.49, Time: 00:02:14 \n",
      "Ep. 260, Timestep 3730,  Ep.Timesteps 14, Score: 17.69, Avg.Score: 16.88, Max.Score: 33.49, Time: 00:02:19 \n",
      "Ep. 270, Timestep 3901,  Ep.Timesteps 15, Score: 18.43, Avg.Score: 17.27, Max.Score: 33.49, Time: 00:02:25 \n",
      "Ep. 280, Timestep 4055,  Ep.Timesteps 8, Score: 12.43, Avg.Score: 17.32, Max.Score: 33.49, Time: 00:02:30 \n",
      "Ep. 290, Timestep 4194,  Ep.Timesteps 12, Score: 16.48, Avg.Score: 16.90, Max.Score: 25.97, Time: 00:02:35 \n",
      "Ep. 300, Timestep 4353,  Ep.Timesteps 18, Score: 17.74, Avg.Score: 17.18, Max.Score: 37.59, Time: 00:02:41 \n",
      "Ep. 310, Timestep 4472,  Ep.Timesteps 17, Score: 19.86, Avg.Score: 16.97, Max.Score: 37.59, Time: 00:02:45 \n",
      "Ep. 320, Timestep 4596,  Ep.Timesteps 11, Score: 16.85, Avg.Score: 16.78, Max.Score: 37.59, Time: 00:02:50 \n",
      "Ep. 330, Timestep 4746,  Ep.Timesteps 17, Score: 17.63, Avg.Score: 16.91, Max.Score: 37.59, Time: 00:02:55 \n",
      "Ep. 340, Timestep 4889,  Ep.Timesteps 24, Score: 22.77, Avg.Score: 16.81, Max.Score: 37.59, Time: 00:03:00 \n",
      "Ep. 350, Timestep 5026,  Ep.Timesteps 9, Score: 13.48, Avg.Score: 16.90, Max.Score: 37.59, Time: 00:03:05 \n",
      "Ep. 360, Timestep 5215,  Ep.Timesteps 16, Score: 16.43, Avg.Score: 17.27, Max.Score: 37.59, Time: 00:03:12 \n",
      "Ep. 370, Timestep 5337,  Ep.Timesteps 12, Score: 17.23, Avg.Score: 17.03, Max.Score: 37.59, Time: 00:03:17 \n",
      "Ep. 380, Timestep 5485,  Ep.Timesteps 20, Score: 18.44, Avg.Score: 17.00, Max.Score: 37.59, Time: 00:03:23 \n",
      "Ep. 390, Timestep 5617,  Ep.Timesteps 13, Score: 16.94, Avg.Score: 16.92, Max.Score: 37.59, Time: 00:03:29 \n",
      "Ep. 400, Timestep 5741,  Ep.Timesteps 8, Score: 12.55, Avg.Score: 16.63, Max.Score: 28.35, Time: 00:03:35 \n",
      "Ep. 410, Timestep 5849,  Ep.Timesteps 10, Score: 13.22, Avg.Score: 16.59, Max.Score: 28.35, Time: 00:03:39 \n",
      "Ep. 420, Timestep 5964,  Ep.Timesteps 10, Score: 15.02, Avg.Score: 16.48, Max.Score: 28.35, Time: 00:03:43 \n",
      "Ep. 430, Timestep 6106,  Ep.Timesteps 12, Score: 13.13, Avg.Score: 16.41, Max.Score: 28.35, Time: 00:03:50 \n",
      "Ep. 440, Timestep 6240,  Ep.Timesteps 11, Score: 15.02, Avg.Score: 16.34, Max.Score: 28.35, Time: 00:03:56 \n",
      "Ep. 450, Timestep 6397,  Ep.Timesteps 15, Score: 16.66, Avg.Score: 16.34, Max.Score: 28.35, Time: 00:04:02 \n",
      "Ep. 460, Timestep 6551,  Ep.Timesteps 14, Score: 16.40, Avg.Score: 16.10, Max.Score: 28.35, Time: 00:04:07 \n",
      "Ep. 470, Timestep 6690,  Ep.Timesteps 13, Score: 13.39, Avg.Score: 16.20, Max.Score: 28.35, Time: 00:04:12 \n",
      "Ep. 480, Timestep 6816,  Ep.Timesteps 20, Score: 19.06, Avg.Score: 16.08, Max.Score: 28.35, Time: 00:04:16 \n",
      "Ep. 490, Timestep 6963,  Ep.Timesteps 30, Score: 26.78, Avg.Score: 16.15, Max.Score: 26.78, Time: 00:04:21 \n",
      "Ep. 500, Timestep 7080,  Ep.Timesteps 7, Score: 11.65, Avg.Score: 16.09, Max.Score: 26.78, Time: 00:04:26 \n",
      "Ep. 510, Timestep 7200,  Ep.Timesteps 11, Score: 14.66, Avg.Score: 16.17, Max.Score: 26.78, Time: 00:04:30 \n",
      "Ep. 520, Timestep 7344,  Ep.Timesteps 15, Score: 17.12, Avg.Score: 16.36, Max.Score: 26.78, Time: 00:04:35 \n",
      "Ep. 530, Timestep 7493,  Ep.Timesteps 8, Score: 13.13, Avg.Score: 16.46, Max.Score: 33.17, Time: 00:04:41 \n",
      "Ep. 540, Timestep 7646,  Ep.Timesteps 10, Score: 13.89, Avg.Score: 16.56, Max.Score: 33.17, Time: 00:04:46 \n",
      "Ep. 550, Timestep 7800,  Ep.Timesteps 8, Score: 11.80, Avg.Score: 16.54, Max.Score: 33.17, Time: 00:04:52 \n",
      "Ep. 560, Timestep 7919,  Ep.Timesteps 9, Score: 13.49, Avg.Score: 16.31, Max.Score: 33.17, Time: 00:04:56 \n",
      "Ep. 570, Timestep 8060,  Ep.Timesteps 14, Score: 14.76, Avg.Score: 16.31, Max.Score: 33.17, Time: 00:05:01 \n",
      "Ep. 580, Timestep 8198,  Ep.Timesteps 20, Score: 19.08, Avg.Score: 16.36, Max.Score: 33.17, Time: 00:05:05 \n",
      "Ep. 590, Timestep 8342,  Ep.Timesteps 12, Score: 15.03, Avg.Score: 16.36, Max.Score: 33.17, Time: 00:05:10 \n",
      "Ep. 600, Timestep 8478,  Ep.Timesteps 14, Score: 16.89, Avg.Score: 16.53, Max.Score: 33.17, Time: 00:05:15 \n",
      "Ep. 610, Timestep 8629,  Ep.Timesteps 11, Score: 14.58, Avg.Score: 16.70, Max.Score: 33.17, Time: 00:05:20 \n",
      "Ep. 620, Timestep 8787,  Ep.Timesteps 11, Score: 14.76, Avg.Score: 16.80, Max.Score: 33.17, Time: 00:05:26 \n",
      "Ep. 630, Timestep 8928,  Ep.Timesteps 13, Score: 16.06, Avg.Score: 16.72, Max.Score: 24.58, Time: 00:05:31 \n",
      "Ep. 640, Timestep 9091,  Ep.Timesteps 21, Score: 17.62, Avg.Score: 16.77, Max.Score: 24.26, Time: 00:05:36 \n",
      "Ep. 650, Timestep 9228,  Ep.Timesteps 8, Score: 12.39, Avg.Score: 16.70, Max.Score: 24.26, Time: 00:05:41 \n",
      "Ep. 660, Timestep 9385,  Ep.Timesteps 30, Score: 25.89, Avg.Score: 16.98, Max.Score: 25.89, Time: 00:05:46 \n",
      "Ep. 670, Timestep 9513,  Ep.Timesteps 8, Score: 13.74, Avg.Score: 16.90, Max.Score: 25.89, Time: 00:05:51 \n",
      "Ep. 680, Timestep 9681,  Ep.Timesteps 11, Score: 13.32, Avg.Score: 17.07, Max.Score: 25.89, Time: 00:05:57 \n",
      "Ep. 690, Timestep 9825,  Ep.Timesteps 18, Score: 19.44, Avg.Score: 17.09, Max.Score: 25.89, Time: 00:06:01 \n",
      "Ep. 700, Timestep 9994,  Ep.Timesteps 10, Score: 15.11, Avg.Score: 17.30, Max.Score: 25.89, Time: 00:06:08 \n",
      "Ep. 710, Timestep 10776,  Ep.Timesteps 76, Score: 74.07, Avg.Score: 23.73, Max.Score: 130.66, Time: 00:06:34 \n",
      "Ep. 720, Timestep 11431,  Ep.Timesteps 36, Score: 36.84, Avg.Score: 28.03, Max.Score: 130.66, Time: 00:07:01 \n",
      "Ep. 730, Timestep 11848,  Ep.Timesteps 31, Score: 29.03, Avg.Score: 30.66, Max.Score: 130.66, Time: 00:07:16 \n",
      "Ep. 740, Timestep 12399,  Ep.Timesteps 74, Score: 95.10, Avg.Score: 35.00, Max.Score: 130.66, Time: 00:07:34 \n",
      "Ep. 750, Timestep 13153,  Ep.Timesteps 62, Score: 81.39, Avg.Score: 41.64, Max.Score: 130.66, Time: 00:08:03 \n",
      "Ep. 760, Timestep 13785,  Ep.Timesteps 66, Score: 81.59, Avg.Score: 47.15, Max.Score: 130.66, Time: 00:08:26 \n",
      "Ep. 770, Timestep 14490,  Ep.Timesteps 62, Score: 80.21, Avg.Score: 53.08, Max.Score: 130.66, Time: 00:08:51 \n",
      "Ep. 780, Timestep 15171,  Ep.Timesteps 83, Score: 50.49, Avg.Score: 56.66, Max.Score: 130.66, Time: 00:09:14 \n",
      "Ep. 790, Timestep 15909,  Ep.Timesteps 82, Score: 87.92, Avg.Score: 61.69, Max.Score: 130.66, Time: 00:09:42 \n",
      "Ep. 800, Timestep 16727,  Ep.Timesteps 103, Score: 74.02, Avg.Score: 67.33, Max.Score: 130.66, Time: 00:10:10 \n",
      "Ep. 810, Timestep 17502,  Ep.Timesteps 67, Score: 65.19, Avg.Score: 64.63, Max.Score: 123.11, Time: 00:10:39 \n",
      "Ep. 820, Timestep 18144,  Ep.Timesteps 62, Score: 68.13, Avg.Score: 64.44, Max.Score: 160.63, Time: 00:11:02 \n",
      "Ep. 830, Timestep 18796,  Ep.Timesteps 75, Score: 88.66, Avg.Score: 67.95, Max.Score: 160.63, Time: 00:11:25 \n",
      "Ep. 840, Timestep 19432,  Ep.Timesteps 68, Score: 78.86, Avg.Score: 69.15, Max.Score: 160.63, Time: 00:11:48 \n",
      "Ep. 850, Timestep 20179,  Ep.Timesteps 51, Score: 68.93, Avg.Score: 69.38, Max.Score: 160.63, Time: 00:12:15 \n",
      "Ep. 860, Timestep 20909,  Ep.Timesteps 106, Score: 115.56, Avg.Score: 70.03, Max.Score: 160.63, Time: 00:12:42 \n",
      "Ep. 870, Timestep 21649,  Ep.Timesteps 66, Score: 77.98, Avg.Score: 71.27, Max.Score: 160.63, Time: 00:13:14 \n",
      "Ep. 880, Timestep 22384,  Ep.Timesteps 102, Score: 116.93, Avg.Score: 74.22, Max.Score: 160.63, Time: 00:13:40 \n",
      "Ep. 890, Timestep 23073,  Ep.Timesteps 134, Score: 89.89, Avg.Score: 75.39, Max.Score: 160.63, Time: 00:14:04 \n",
      "Ep. 900, Timestep 23847,  Ep.Timesteps 58, Score: 25.64, Avg.Score: 76.81, Max.Score: 160.63, Time: 00:14:34 \n",
      "Ep. 910, Timestep 24515,  Ep.Timesteps 75, Score: 95.52, Avg.Score: 78.87, Max.Score: 160.63, Time: 00:14:57 \n",
      "Ep. 920, Timestep 25289,  Ep.Timesteps 59, Score: 74.24, Avg.Score: 82.44, Max.Score: 142.77, Time: 00:15:25 \n",
      "Ep. 930, Timestep 25999,  Ep.Timesteps 75, Score: 85.04, Avg.Score: 83.17, Max.Score: 142.77, Time: 00:15:50 \n",
      "Ep. 940, Timestep 26630,  Ep.Timesteps 71, Score: 83.99, Avg.Score: 83.10, Max.Score: 142.77, Time: 00:16:12 \n",
      "Ep. 950, Timestep 27254,  Ep.Timesteps 54, Score: 64.24, Avg.Score: 82.39, Max.Score: 142.77, Time: 00:16:36 \n",
      "Ep. 960, Timestep 28002,  Ep.Timesteps 103, Score: 33.61, Avg.Score: 82.86, Max.Score: 142.77, Time: 00:17:01 \n",
      "Ep. 970, Timestep 28593,  Ep.Timesteps 33, Score: 30.24, Avg.Score: 79.66, Max.Score: 142.77, Time: 00:17:25 \n",
      "Ep. 980, Timestep 29199,  Ep.Timesteps 54, Score: 73.43, Avg.Score: 78.56, Max.Score: 139.53, Time: 00:17:47 \n",
      "Ep. 990, Timestep 29836,  Ep.Timesteps 86, Score: 99.38, Avg.Score: 78.30, Max.Score: 139.53, Time: 00:18:09 \n",
      "Ep. 1000, Timestep 30521,  Ep.Timesteps 58, Score: 71.60, Avg.Score: 76.23, Max.Score: 139.53, Time: 00:18:34 \n",
      "Ep. 1010, Timestep 31223,  Ep.Timesteps 101, Score: 130.86, Avg.Score: 77.03, Max.Score: 139.53, Time: 00:18:57 \n",
      "Ep. 1020, Timestep 32029,  Ep.Timesteps 81, Score: 99.78, Avg.Score: 77.53, Max.Score: 153.17, Time: 00:19:27 \n",
      "Ep. 1030, Timestep 32951,  Ep.Timesteps 132, Score: 144.24, Avg.Score: 78.14, Max.Score: 185.40, Time: 00:19:59 \n",
      "Ep. 1040, Timestep 34275,  Ep.Timesteps 199, Score: 204.91, Avg.Score: 84.99, Max.Score: 255.66, Time: 00:20:45 \n",
      "Ep. 1050, Timestep 36201,  Ep.Timesteps 64, Score: 73.29, Avg.Score: 95.56, Max.Score: 789.46, Time: 00:21:59 \n",
      "Ep. 1060, Timestep 36923,  Ep.Timesteps 89, Score: 106.31, Avg.Score: 95.87, Max.Score: 789.46, Time: 00:22:25 \n",
      "Ep. 1070, Timestep 37701,  Ep.Timesteps 61, Score: 72.20, Avg.Score: 99.12, Max.Score: 789.46, Time: 00:22:53 \n",
      "Ep. 1080, Timestep 39553,  Ep.Timesteps 96, Score: 110.48, Avg.Score: 109.38, Max.Score: 868.70, Time: 00:23:59 \n",
      "Ep. 1090, Timestep 40616,  Ep.Timesteps 93, Score: 12.73, Avg.Score: 109.66, Max.Score: 868.70, Time: 00:24:41 \n",
      "Ep. 1100, Timestep 43237,  Ep.Timesteps 95, Score: 101.56, Avg.Score: 125.02, Max.Score: 868.70, Time: 00:26:18 \n",
      "Ep. 1110, Timestep 46278,  Ep.Timesteps 1000, Score: 803.87, Avg.Score: 141.85, Max.Score: 868.70, Time: 00:27:42 \n",
      "Ep. 1120, Timestep 47010,  Ep.Timesteps 73, Score: 73.85, Avg.Score: 140.45, Max.Score: 868.70, Time: 00:28:54 \n",
      "Ep. 1130, Timestep 47827,  Ep.Timesteps 68, Score: 86.84, Avg.Score: 140.91, Max.Score: 868.70, Time: 00:29:30 \n",
      "Ep. 1140, Timestep 49584,  Ep.Timesteps 73, Score: 83.34, Avg.Score: 143.50, Max.Score: 868.70, Time: 00:30:59 \n",
      "Ep. 1150, Timestep 50760,  Ep.Timesteps 83, Score: 115.58, Avg.Score: 137.21, Max.Score: 868.70, Time: 00:31:54 \n",
      "Ep. 1160, Timestep 53733,  Ep.Timesteps 119, Score: 151.74, Avg.Score: 151.73, Max.Score: 868.70, Time: 00:33:38 \n",
      "Ep. 1170, Timestep 55921,  Ep.Timesteps 1000, Score: 745.95, Avg.Score: 163.63, Max.Score: 868.70, Time: 00:34:39 \n",
      "Ep. 1180, Timestep 58291,  Ep.Timesteps 218, Score: 223.12, Avg.Score: 169.18, Max.Score: 811.57, Time: 00:36:44 \n",
      "Ep. 1190, Timestep 64303,  Ep.Timesteps 87, Score: 92.04, Avg.Score: 212.93, Max.Score: 903.67, Time: 00:40:31 \n",
      "Ep. 1200, Timestep 67061,  Ep.Timesteps 109, Score: 100.20, Avg.Score: 214.62, Max.Score: 903.67, Time: 00:42:15 \n",
      "Ep. 1210, Timestep 70268,  Ep.Timesteps 173, Score: 216.83, Avg.Score: 216.33, Max.Score: 903.67, Time: 00:44:21 \n",
      "Ep. 1220, Timestep 74709,  Ep.Timesteps 1000, Score: 695.22, Avg.Score: 246.63, Max.Score: 903.67, Time: 00:46:33 \n",
      "Ep. 1230, Timestep 77302,  Ep.Timesteps 41, Score: 6.42, Avg.Score: 256.07, Max.Score: 903.67, Time: 00:48:45 \n",
      "Ep. 1240, Timestep 78961,  Ep.Timesteps 47, Score: -0.45, Avg.Score: 250.72, Max.Score: 903.67, Time: 00:49:49 \n",
      "Ep. 1250, Timestep 80901,  Ep.Timesteps 29, Score: 23.98, Avg.Score: 252.54, Max.Score: 903.67, Time: 00:51:00 \n",
      "Ep. 1260, Timestep 84094,  Ep.Timesteps 114, Score: 118.73, Avg.Score: 256.37, Max.Score: 903.67, Time: 00:52:54 \n",
      "Ep. 1270, Timestep 90149,  Ep.Timesteps 1000, Score: 808.75, Avg.Score: 283.94, Max.Score: 903.67, Time: 00:56:05 \n",
      "Ep. 1280, Timestep 93256,  Ep.Timesteps 143, Score: 125.49, Avg.Score: 287.01, Max.Score: 903.67, Time: 00:58:25 \n",
      "Ep. 1290, Timestep 93969,  Ep.Timesteps 62, Score: 77.26, Avg.Score: 242.68, Max.Score: 868.29, Time: 00:58:53 \n",
      "Ep. 1300, Timestep 95798,  Ep.Timesteps 57, Score: 71.13, Avg.Score: 236.30, Max.Score: 816.36, Time: 00:59:58 \n",
      "Ep. 1310, Timestep 97049,  Ep.Timesteps 93, Score: 124.06, Avg.Score: 224.80, Max.Score: 816.36, Time: 01:00:41 \n",
      "Ep. 1320, Timestep 98107,  Ep.Timesteps 80, Score: 87.65, Avg.Score: 197.30, Max.Score: 816.36, Time: 01:01:19 \n",
      "Ep. 1330, Timestep 100115,  Ep.Timesteps 1000, Score: 916.84, Avg.Score: 199.85, Max.Score: 916.84, Time: 01:02:00 \n",
      "Ep. 1340, Timestep 103233,  Ep.Timesteps 80, Score: 90.97, Avg.Score: 226.75, Max.Score: 1358.45, Time: 01:04:20 \n",
      "Ep. 1350, Timestep 105485,  Ep.Timesteps 110, Score: 119.24, Avg.Score: 235.07, Max.Score: 1358.45, Time: 01:05:37 \n",
      "Ep. 1360, Timestep 107559,  Ep.Timesteps 96, Score: 113.33, Avg.Score: 226.82, Max.Score: 1358.45, Time: 01:06:58 \n",
      "Ep. 1370, Timestep 110961,  Ep.Timesteps 216, Score: 217.09, Avg.Score: 208.14, Max.Score: 1358.45, Time: 01:09:28 \n",
      "Ep. 1380, Timestep 114219,  Ep.Timesteps 87, Score: 90.22, Avg.Score: 210.98, Max.Score: 1358.45, Time: 01:11:25 \n",
      "Ep. 1390, Timestep 117989,  Ep.Timesteps 102, Score: 117.76, Avg.Score: 235.51, Max.Score: 1358.45, Time: 01:13:36 \n",
      "Ep. 1400, Timestep 118721,  Ep.Timesteps 79, Score: 4.46, Avg.Score: 224.65, Max.Score: 1358.45, Time: 01:14:02 \n",
      "Ep. 1410, Timestep 120238,  Ep.Timesteps 1000, Score: 798.41, Avg.Score: 221.41, Max.Score: 1358.45, Time: 01:14:27 \n",
      "Ep. 1420, Timestep 121060,  Ep.Timesteps 47, Score: 53.38, Avg.Score: 219.30, Max.Score: 1358.45, Time: 01:15:31 \n",
      "Ep. 1430, Timestep 121735,  Ep.Timesteps 84, Score: 90.84, Avg.Score: 204.30, Max.Score: 1358.45, Time: 01:15:53 \n",
      "Ep. 1440, Timestep 122528,  Ep.Timesteps 95, Score: 21.62, Avg.Score: 171.92, Max.Score: 916.39, Time: 01:16:21 \n",
      "Ep. 1450, Timestep 126712,  Ep.Timesteps 1000, Score: 804.80, Avg.Score: 184.31, Max.Score: 856.93, Time: 01:18:20 \n",
      "Ep. 1460, Timestep 134617,  Ep.Timesteps 160, Score: 183.06, Avg.Score: 235.57, Max.Score: 956.14, Time: 01:23:41 \n",
      "Ep. 1470, Timestep 138689,  Ep.Timesteps 215, Score: 272.68, Avg.Score: 244.02, Max.Score: 956.14, Time: 01:26:10 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 1480, Timestep 145517,  Ep.Timesteps 1000, Score: 1389.54, Avg.Score: 291.56, Max.Score: 1389.54, Time: 01:29:46 \n",
      "Ep. 1490, Timestep 151494,  Ep.Timesteps 585, Score: 720.36, Avg.Score: 324.66, Max.Score: 1389.54, Time: 01:33:26 \n",
      "Ep. 1500, Timestep 157246,  Ep.Timesteps 1000, Score: 829.30, Avg.Score: 369.39, Max.Score: 1389.54, Time: 01:36:31 \n",
      "Ep. 1510, Timestep 160266,  Ep.Timesteps 193, Score: 235.93, Avg.Score: 388.90, Max.Score: 1389.54, Time: 01:38:42 \n",
      "Ep. 1520, Timestep 165260,  Ep.Timesteps 801, Score: 760.39, Avg.Score: 424.22, Max.Score: 1389.54, Time: 01:41:14 \n",
      "Ep. 1530, Timestep 173172,  Ep.Timesteps 1000, Score: 1232.69, Avg.Score: 509.70, Max.Score: 1389.54, Time: 01:45:39 \n",
      "Ep. 1540, Timestep 181945,  Ep.Timesteps 1000, Score: 829.50, Avg.Score: 609.87, Max.Score: 1389.54, Time: 01:50:41 \n",
      "Ep. 1550, Timestep 189716,  Ep.Timesteps 1000, Score: 1196.66, Avg.Score: 664.49, Max.Score: 1418.66, Time: 01:55:08 \n",
      "Ep. 1560, Timestep 196200,  Ep.Timesteps 123, Score: 144.87, Avg.Score: 675.33, Max.Score: 1418.66, Time: 01:59:16 \n",
      "Ep. 1570, Timestep 203534,  Ep.Timesteps 135, Score: 183.77, Avg.Score: 726.97, Max.Score: 1418.66, Time: 02:03:55 \n",
      "Ep. 1580, Timestep 210643,  Ep.Timesteps 1000, Score: 1398.96, Avg.Score: 726.89, Max.Score: 1418.66, Time: 02:07:31 \n",
      "Ep. 1590, Timestep 216967,  Ep.Timesteps 436, Score: 609.64, Avg.Score: 735.78, Max.Score: 1418.66, Time: 02:11:24 \n",
      "Ep. 1600, Timestep 220196,  Ep.Timesteps 456, Score: 606.20, Avg.Score: 727.80, Max.Score: 1418.66, Time: 02:13:14 \n",
      "Ep. 1610, Timestep 227152,  Ep.Timesteps 1000, Score: 1259.91, Avg.Score: 789.65, Max.Score: 1462.37, Time: 02:16:55 \n",
      "Ep. 1620, Timestep 235423,  Ep.Timesteps 225, Score: 282.59, Avg.Score: 858.41, Max.Score: 1462.37, Time: 02:22:08 \n",
      "Ep. 1630, Timestep 244247,  Ep.Timesteps 1000, Score: 1287.35, Avg.Score: 869.55, Max.Score: 1462.37, Time: 02:26:45 \n",
      "Ep. 1640, Timestep 251551,  Ep.Timesteps 1000, Score: 849.56, Avg.Score: 852.08, Max.Score: 1462.37, Time: 02:30:54 \n",
      "Ep. 1650, Timestep 260410,  Ep.Timesteps 524, Score: 692.85, Avg.Score: 872.27, Max.Score: 1462.37, Time: 02:36:12 \n",
      "Ep. 1660, Timestep 266072,  Ep.Timesteps 1000, Score: 1260.95, Avg.Score: 867.08, Max.Score: 1462.37, Time: 02:39:09 \n",
      "Ep. 1670, Timestep 274267,  Ep.Timesteps 1000, Score: 1403.92, Avg.Score: 884.20, Max.Score: 1462.37, Time: 02:43:54 \n",
      "Ep. 1680, Timestep 282521,  Ep.Timesteps 1000, Score: 1299.87, Avg.Score: 909.09, Max.Score: 1462.37, Time: 02:48:43 \n",
      "Ep. 1690, Timestep 292521,  Ep.Timesteps 1000, Score: 1396.75, Avg.Score: 969.16, Max.Score: 1462.37, Time: 02:55:29 \n",
      "Ep. 1700, Timestep 301504,  Ep.Timesteps 443, Score: 627.76, Avg.Score: 1051.66, Max.Score: 1462.37, Time: 03:01:21 \n",
      "Ep. 1710, Timestep 308341,  Ep.Timesteps 1000, Score: 1537.81, Avg.Score: 1057.85, Max.Score: 1537.81, Time: 03:05:17 \n",
      "Ep. 1720, Timestep 318341,  Ep.Timesteps 1000, Score: 1460.21, Avg.Score: 1091.82, Max.Score: 1628.52, Time: 03:11:18 \n",
      "Ep. 1730, Timestep 328341,  Ep.Timesteps 1000, Score: 1384.25, Avg.Score: 1126.23, Max.Score: 1628.52, Time: 03:17:21 \n",
      "Ep. 1740, Timestep 338341,  Ep.Timesteps 1000, Score: 1360.00, Avg.Score: 1176.23, Max.Score: 1628.52, Time: 03:23:25 \n",
      "Ep. 1750, Timestep 347402,  Ep.Timesteps 1000, Score: 1502.25, Avg.Score: 1196.51, Max.Score: 1628.52, Time: 03:28:55 \n",
      "Ep. 1760, Timestep 355087,  Ep.Timesteps 1000, Score: 1467.90, Avg.Score: 1231.32, Max.Score: 1628.52, Time: 03:33:34 \n",
      "Ep. 1770, Timestep 364337,  Ep.Timesteps 1000, Score: 1570.96, Avg.Score: 1264.60, Max.Score: 1628.52, Time: 03:39:09 \n",
      "Ep. 1780, Timestep 372665,  Ep.Timesteps 1000, Score: 1443.81, Avg.Score: 1289.47, Max.Score: 1628.52, Time: 03:44:12 \n",
      "Ep. 1790, Timestep 381360,  Ep.Timesteps 1000, Score: 1549.60, Avg.Score: 1289.59, Max.Score: 1628.52, Time: 03:49:28 \n",
      "Ep. 1800, Timestep 390497,  Ep.Timesteps 1000, Score: 1731.78, Avg.Score: 1309.78, Max.Score: 1731.78, Time: 03:54:59 \n",
      "Ep. 1810, Timestep 399653,  Ep.Timesteps 1000, Score: 1570.79, Avg.Score: 1353.12, Max.Score: 1731.78, Time: 04:00:21 \n",
      "Ep. 1820, Timestep 406654,  Ep.Timesteps 846, Score: 1409.79, Avg.Score: 1319.42, Max.Score: 1731.78, Time: 04:04:36 \n",
      "Ep. 1830, Timestep 416654,  Ep.Timesteps 1000, Score: 1587.20, Avg.Score: 1341.59, Max.Score: 1731.78, Time: 04:10:45 \n",
      "Ep. 1840, Timestep 424969,  Ep.Timesteps 1000, Score: 1616.81, Avg.Score: 1335.67, Max.Score: 1731.78, Time: 04:15:47 \n",
      "Ep. 1850, Timestep 431979,  Ep.Timesteps 1000, Score: 1694.78, Avg.Score: 1320.25, Max.Score: 1731.78, Time: 04:20:01 \n",
      "Ep. 1860, Timestep 436831,  Ep.Timesteps 1000, Score: 1677.02, Avg.Score: 1287.80, Max.Score: 1731.78, Time: 04:22:56 \n",
      "Ep. 1870, Timestep 444005,  Ep.Timesteps 294, Score: 454.64, Avg.Score: 1267.68, Max.Score: 1763.93, Time: 04:27:36 \n",
      "Ep. 1880, Timestep 450210,  Ep.Timesteps 201, Score: 321.24, Avg.Score: 1236.78, Max.Score: 1763.93, Time: 04:31:21 \n",
      "Ep. 1890, Timestep 458171,  Ep.Timesteps 1000, Score: 1666.18, Avg.Score: 1224.73, Max.Score: 1763.93, Time: 04:35:40 \n",
      "Ep. 1900, Timestep 467277,  Ep.Timesteps 1000, Score: 1688.27, Avg.Score: 1227.45, Max.Score: 1763.93, Time: 04:41:12 \n",
      "Ep. 1910, Timestep 477277,  Ep.Timesteps 1000, Score: 1625.43, Avg.Score: 1246.36, Max.Score: 1763.93, Time: 04:47:30 \n",
      "Ep. 1920, Timestep 482503,  Ep.Timesteps 454, Score: 728.77, Avg.Score: 1217.72, Max.Score: 1763.93, Time: 04:51:10 \n",
      "Ep. 1930, Timestep 489664,  Ep.Timesteps 169, Score: 252.95, Avg.Score: 1177.35, Max.Score: 1763.93, Time: 04:55:57 \n",
      "Ep. 1940, Timestep 496403,  Ep.Timesteps 91, Score: 131.71, Avg.Score: 1155.78, Max.Score: 1763.93, Time: 04:59:59 \n",
      "Ep. 1950, Timestep 503036,  Ep.Timesteps 1000, Score: 1576.02, Avg.Score: 1146.63, Max.Score: 1763.93, Time: 05:03:18 \n",
      "Ep. 1960, Timestep 508822,  Ep.Timesteps 1000, Score: 1645.26, Avg.Score: 1161.76, Max.Score: 1763.93, Time: 05:06:40 \n",
      "Ep. 1970, Timestep 515805,  Ep.Timesteps 52, Score: 76.06, Avg.Score: 1160.47, Max.Score: 1748.68, Time: 05:11:13 \n",
      "Ep. 1980, Timestep 520400,  Ep.Timesteps 1000, Score: 1578.52, Avg.Score: 1141.74, Max.Score: 1797.89, Time: 05:13:22 \n",
      "Ep. 1990, Timestep 530400,  Ep.Timesteps 1000, Score: 1722.75, Avg.Score: 1189.44, Max.Score: 1797.89, Time: 05:19:06 \n",
      "Ep. 2000, Timestep 538612,  Ep.Timesteps 143, Score: 233.52, Avg.Score: 1181.64, Max.Score: 1799.71, Time: 05:24:15 \n",
      "Ep. 2010, Timestep 546721,  Ep.Timesteps 1000, Score: 1650.37, Avg.Score: 1155.41, Max.Score: 1799.71, Time: 05:28:30 \n",
      "Ep. 2020, Timestep 556592,  Ep.Timesteps 1000, Score: 1682.05, Avg.Score: 1241.60, Max.Score: 1799.71, Time: 05:34:07 \n",
      "Ep. 2030, Timestep 565166,  Ep.Timesteps 1000, Score: 1715.45, Avg.Score: 1265.52, Max.Score: 1799.71, Time: 05:39:03 \n",
      "Ep. 2040, Timestep 575166,  Ep.Timesteps 1000, Score: 1691.46, Avg.Score: 1327.23, Max.Score: 1834.13, Time: 05:44:47 \n",
      "Ep. 2050, Timestep 585166,  Ep.Timesteps 1000, Score: 1652.39, Avg.Score: 1393.42, Max.Score: 1834.13, Time: 05:50:32 \n",
      "Ep. 2060, Timestep 593493,  Ep.Timesteps 1000, Score: 1563.60, Avg.Score: 1437.78, Max.Score: 1834.13, Time: 05:55:24 \n",
      "Ep. 2070, Timestep 601771,  Ep.Timesteps 117, Score: 178.65, Avg.Score: 1457.20, Max.Score: 1834.13, Time: 06:00:37 \n",
      "Ep. 2080, Timestep 602605,  Ep.Timesteps 73, Score: 112.01, Avg.Score: 1392.57, Max.Score: 1834.13, Time: 06:01:07 \n",
      "Ep. 2090, Timestep 605047,  Ep.Timesteps 470, Score: 778.26, Avg.Score: 1262.42, Max.Score: 1834.13, Time: 06:02:18 \n",
      "Ep. 2100, Timestep 612788,  Ep.Timesteps 1000, Score: 1816.40, Avg.Score: 1235.26, Max.Score: 1834.13, Time: 06:06:42 \n",
      "Ep. 2110, Timestep 622089,  Ep.Timesteps 1000, Score: 1530.55, Avg.Score: 1252.15, Max.Score: 1834.13, Time: 06:12:06 \n",
      "Ep. 2120, Timestep 630454,  Ep.Timesteps 1000, Score: 1510.27, Avg.Score: 1209.28, Max.Score: 1834.13, Time: 06:16:59 \n",
      "Ep. 2130, Timestep 640454,  Ep.Timesteps 1000, Score: 1586.29, Avg.Score: 1221.28, Max.Score: 1834.13, Time: 06:22:54 \n",
      "Ep. 2140, Timestep 648742,  Ep.Timesteps 1000, Score: 1520.55, Avg.Score: 1178.34, Max.Score: 1816.40, Time: 06:27:39 \n",
      "Ep. 2150, Timestep 656909,  Ep.Timesteps 1000, Score: 1476.93, Avg.Score: 1128.87, Max.Score: 1816.40, Time: 06:32:21 \n",
      "Ep. 2160, Timestep 666158,  Ep.Timesteps 1000, Score: 1587.88, Avg.Score: 1131.62, Max.Score: 1816.40, Time: 06:37:45 \n",
      "Ep. 2170, Timestep 676158,  Ep.Timesteps 1000, Score: 1710.67, Avg.Score: 1161.42, Max.Score: 1816.40, Time: 06:43:36 \n",
      "Ep. 2180, Timestep 686158,  Ep.Timesteps 1000, Score: 1753.66, Avg.Score: 1323.26, Max.Score: 1816.40, Time: 06:49:27 \n",
      "Ep. 2190, Timestep 695814,  Ep.Timesteps 1000, Score: 1661.21, Avg.Score: 1450.73, Max.Score: 1816.40, Time: 06:55:06 \n",
      "Ep. 2200, Timestep 705715,  Ep.Timesteps 1000, Score: 1765.59, Avg.Score: 1505.92, Max.Score: 1783.16, Time: 07:00:48 \n",
      "Ep. 2210, Timestep 715715,  Ep.Timesteps 1000, Score: 1725.74, Avg.Score: 1528.44, Max.Score: 1816.43, Time: 07:06:46 \n",
      "Ep. 2220, Timestep 720711,  Ep.Timesteps 1000, Score: 1784.46, Avg.Score: 1488.28, Max.Score: 1816.43, Time: 07:09:37 \n",
      "Ep. 2230, Timestep 730652,  Ep.Timesteps 1000, Score: 1813.91, Avg.Score: 1502.71, Max.Score: 1816.43, Time: 07:15:58 \n",
      "Ep. 2240, Timestep 740158,  Ep.Timesteps 1000, Score: 1641.03, Avg.Score: 1539.79, Max.Score: 1867.60, Time: 07:21:42 \n",
      "Ep. 2250, Timestep 750158,  Ep.Timesteps 1000, Score: 1716.45, Avg.Score: 1592.13, Max.Score: 1867.60, Time: 07:27:50 \n",
      "Ep. 2260, Timestep 759648,  Ep.Timesteps 1000, Score: 1805.24, Avg.Score: 1615.12, Max.Score: 1867.60, Time: 07:33:19 \n",
      "Ep. 2270, Timestep 764834,  Ep.Timesteps 1000, Score: 1689.52, Avg.Score: 1534.81, Max.Score: 1867.60, Time: 07:36:24 \n",
      "Ep. 2280, Timestep 774834,  Ep.Timesteps 1000, Score: 1591.32, Avg.Score: 1490.36, Max.Score: 1867.60, Time: 07:42:09 \n",
      "Ep. 2290, Timestep 784038,  Ep.Timesteps 1000, Score: 1710.74, Avg.Score: 1467.96, Max.Score: 1867.60, Time: 07:47:22 \n",
      "Ep. 2300, Timestep 793708,  Ep.Timesteps 1000, Score: 1751.40, Avg.Score: 1470.33, Max.Score: 1913.02, Time: 07:53:01 \n",
      "Ep. 2310, Timestep 803708,  Ep.Timesteps 1000, Score: 1806.05, Avg.Score: 1467.88, Max.Score: 1913.02, Time: 07:58:45 \n",
      "Ep. 2320, Timestep 813146,  Ep.Timesteps 1000, Score: 1785.10, Avg.Score: 1545.85, Max.Score: 1913.02, Time: 08:04:13 \n",
      "Ep. 2330, Timestep 822448,  Ep.Timesteps 1000, Score: 1794.94, Avg.Score: 1542.53, Max.Score: 1913.02, Time: 08:09:32 \n",
      "Ep. 2340, Timestep 831958,  Ep.Timesteps 1000, Score: 1781.94, Avg.Score: 1543.21, Max.Score: 1913.02, Time: 08:14:56 \n",
      "Ep. 2350, Timestep 841958,  Ep.Timesteps 1000, Score: 1717.98, Avg.Score: 1544.42, Max.Score: 1913.02, Time: 08:20:42 \n",
      "Ep. 2360, Timestep 850097,  Ep.Timesteps 1000, Score: 1683.56, Avg.Score: 1522.96, Max.Score: 1913.02, Time: 08:25:22 \n",
      "Ep. 2370, Timestep 857212,  Ep.Timesteps 1000, Score: 1801.09, Avg.Score: 1557.55, Max.Score: 1913.02, Time: 08:29:35 \n",
      "Ep. 2380, Timestep 867212,  Ep.Timesteps 1000, Score: 1849.04, Avg.Score: 1607.03, Max.Score: 1913.02, Time: 08:35:35 \n",
      "Ep. 2390, Timestep 876257,  Ep.Timesteps 45, Score: 57.58, Avg.Score: 1626.35, Max.Score: 1913.02, Time: 08:41:13 \n",
      "Ep. 2400, Timestep 885427,  Ep.Timesteps 1000, Score: 1824.38, Avg.Score: 1615.61, Max.Score: 1877.03, Time: 08:45:56 \n",
      "Ep. 2410, Timestep 895427,  Ep.Timesteps 1000, Score: 1758.49, Avg.Score: 1622.45, Max.Score: 1877.03, Time: 08:51:43 \n",
      "Ep. 2420, Timestep 902969,  Ep.Timesteps 1000, Score: 1879.38, Avg.Score: 1593.53, Max.Score: 1879.38, Time: 08:55:58 \n",
      "Ep. 2430, Timestep 912005,  Ep.Timesteps 1000, Score: 1687.45, Avg.Score: 1588.72, Max.Score: 1879.38, Time: 09:01:05 \n",
      "Ep. 2440, Timestep 921094,  Ep.Timesteps 1000, Score: 1828.47, Avg.Score: 1584.50, Max.Score: 1881.21, Time: 09:06:21 \n",
      "Ep. 2450, Timestep 930214,  Ep.Timesteps 1000, Score: 1708.18, Avg.Score: 1568.94, Max.Score: 1910.88, Time: 09:11:31 \n",
      "Ep. 2460, Timestep 940214,  Ep.Timesteps 1000, Score: 1876.28, Avg.Score: 1606.32, Max.Score: 1910.88, Time: 09:17:11 \n",
      "Ep. 2470, Timestep 950214,  Ep.Timesteps 1000, Score: 1828.70, Avg.Score: 1665.67, Max.Score: 1910.88, Time: 09:22:58 \n",
      "Ep. 2480, Timestep 959321,  Ep.Timesteps 1000, Score: 1845.67, Avg.Score: 1655.22, Max.Score: 1910.88, Time: 09:28:08 \n",
      "Ep. 2490, Timestep 968399,  Ep.Timesteps 1000, Score: 1822.43, Avg.Score: 1654.73, Max.Score: 1910.88, Time: 09:33:16 \n",
      "Ep. 2500, Timestep 975836,  Ep.Timesteps 1000, Score: 1755.89, Avg.Score: 1622.08, Max.Score: 1910.88, Time: 09:37:37 \n",
      "Ep. 2510, Timestep 985836,  Ep.Timesteps 1000, Score: 1852.10, Avg.Score: 1623.75, Max.Score: 1910.88, Time: 09:43:18 \n",
      "Ep. 2520, Timestep 994989,  Ep.Timesteps 1000, Score: 1816.92, Avg.Score: 1650.47, Max.Score: 1910.88, Time: 09:48:30 \n",
      "Ep. 2530, Timestep 1004989,  Ep.Timesteps 1000, Score: 1774.33, Avg.Score: 1667.60, Max.Score: 1910.88, Time: 09:54:20 \n",
      "Ep. 2540, Timestep 1014490,  Ep.Timesteps 1000, Score: 1803.50, Avg.Score: 1676.81, Max.Score: 1910.88, Time: 09:59:45 \n",
      "Ep. 2550, Timestep 1023635,  Ep.Timesteps 1000, Score: 1812.42, Avg.Score: 1679.36, Max.Score: 1903.95, Time: 10:05:22 \n",
      "Ep. 2560, Timestep 1031400,  Ep.Timesteps 692, Score: 1215.26, Avg.Score: 1640.21, Max.Score: 1903.95, Time: 10:09:59 \n",
      "Ep. 2570, Timestep 1040492,  Ep.Timesteps 92, Score: 146.90, Avg.Score: 1623.76, Max.Score: 1907.53, Time: 10:15:28 \n",
      "Ep. 2580, Timestep 1049643,  Ep.Timesteps 1000, Score: 1758.74, Avg.Score: 1620.75, Max.Score: 1907.53, Time: 10:20:17 \n",
      "Ep. 2590, Timestep 1058624,  Ep.Timesteps 1000, Score: 1819.23, Avg.Score: 1620.90, Max.Score: 1907.53, Time: 10:25:25 \n",
      "Ep. 2600, Timestep 1066857,  Ep.Timesteps 1000, Score: 1826.71, Avg.Score: 1643.63, Max.Score: 1942.84, Time: 10:30:06 \n",
      "Ep. 2610, Timestep 1074408,  Ep.Timesteps 165, Score: 290.19, Avg.Score: 1599.80, Max.Score: 1942.84, Time: 10:34:56 \n",
      "Ep. 2620, Timestep 1083670,  Ep.Timesteps 1000, Score: 1801.76, Avg.Score: 1606.58, Max.Score: 1950.32, Time: 10:39:50 \n",
      "Ep. 2630, Timestep 1093059,  Ep.Timesteps 1000, Score: 1845.37, Avg.Score: 1599.63, Max.Score: 1950.32, Time: 10:45:11 \n",
      "Ep. 2640, Timestep 1102260,  Ep.Timesteps 1000, Score: 1742.63, Avg.Score: 1591.06, Max.Score: 1950.32, Time: 10:50:29 \n",
      "Ep. 2650, Timestep 1109486,  Ep.Timesteps 1000, Score: 1765.39, Avg.Score: 1557.58, Max.Score: 1950.32, Time: 10:54:39 \n",
      "Ep. 2660, Timestep 1118645,  Ep.Timesteps 1000, Score: 1781.39, Avg.Score: 1582.53, Max.Score: 1950.32, Time: 10:59:53 \n",
      "Ep. 2670, Timestep 1128645,  Ep.Timesteps 1000, Score: 1874.10, Avg.Score: 1606.95, Max.Score: 1968.28, Time: 11:05:43 \n",
      "Ep. 2680, Timestep 1138350,  Ep.Timesteps 1000, Score: 1942.69, Avg.Score: 1612.30, Max.Score: 1968.28, Time: 11:11:15 \n",
      "Ep. 2690, Timestep 1144956,  Ep.Timesteps 1000, Score: 1849.13, Avg.Score: 1566.84, Max.Score: 1968.28, Time: 11:15:02 \n",
      "Ep. 2700, Timestep 1147300,  Ep.Timesteps 34, Score: 31.44, Avg.Score: 1453.84, Max.Score: 1968.28, Time: 11:16:53 \n",
      "Ep. 2710, Timestep 1151337,  Ep.Timesteps 1000, Score: 1659.08, Avg.Score: 1382.33, Max.Score: 1968.28, Time: 11:18:42 \n",
      "Ep. 2720, Timestep 1151954,  Ep.Timesteps 55, Score: 46.81, Avg.Score: 1216.05, Max.Score: 1968.28, Time: 11:19:37 \n",
      "Ep. 2730, Timestep 1152540,  Ep.Timesteps 72, Score: 64.47, Avg.Score: 1048.69, Max.Score: 1968.28, Time: 11:19:57 \n",
      "Ep. 2740, Timestep 1153121,  Ep.Timesteps 50, Score: 36.72, Avg.Score: 889.13, Max.Score: 1968.28, Time: 11:20:18 \n",
      "Ep. 2750, Timestep 1153736,  Ep.Timesteps 63, Score: 59.74, Avg.Score: 765.03, Max.Score: 1968.28, Time: 11:20:39 \n",
      "Ep. 2760, Timestep 1154338,  Ep.Timesteps 52, Score: 51.22, Avg.Score: 604.96, Max.Score: 1968.28, Time: 11:21:00 \n",
      "Ep. 2770, Timestep 1155007,  Ep.Timesteps 95, Score: 118.81, Avg.Score: 421.53, Max.Score: 1942.69, Time: 11:21:22 \n",
      "Ep. 2780, Timestep 1164048,  Ep.Timesteps 41, Score: 25.43, Avg.Score: 379.55, Max.Score: 1905.22, Time: 11:26:34 \n",
      "Ep. 2790, Timestep 1167648,  Ep.Timesteps 291, Score: 403.35, Avg.Score: 312.44, Max.Score: 1779.69, Time: 11:28:30 \n",
      "Ep. 2800, Timestep 1173560,  Ep.Timesteps 1000, Score: 1624.69, Avg.Score: 367.36, Max.Score: 1712.80, Time: 11:31:30 \n",
      "Ep. 2810, Timestep 1182475,  Ep.Timesteps 1000, Score: 1745.09, Avg.Score: 449.78, Max.Score: 1805.60, Time: 11:36:42 \n",
      "Ep. 2820, Timestep 1192475,  Ep.Timesteps 1000, Score: 1658.84, Avg.Score: 613.28, Max.Score: 1805.60, Time: 11:42:24 \n",
      "Ep. 2830, Timestep 1202475,  Ep.Timesteps 1000, Score: 1718.57, Avg.Score: 786.38, Max.Score: 1849.59, Time: 11:48:06 \n",
      "Ep. 2840, Timestep 1211314,  Ep.Timesteps 1000, Score: 1816.10, Avg.Score: 938.21, Max.Score: 1864.45, Time: 11:53:16 \n",
      "Ep. 2850, Timestep 1221314,  Ep.Timesteps 1000, Score: 1718.23, Avg.Score: 1113.51, Max.Score: 1867.46, Time: 11:58:59 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 2860, Timestep 1230659,  Ep.Timesteps 345, Score: 608.42, Avg.Score: 1279.63, Max.Score: 1925.59, Time: 12:04:44 \n",
      "Ep. 2870, Timestep 1240215,  Ep.Timesteps 1000, Score: 1848.46, Avg.Score: 1447.95, Max.Score: 1970.49, Time: 12:09:52 \n",
      "Ep. 2880, Timestep 1246892,  Ep.Timesteps 581, Score: 1045.23, Avg.Score: 1432.06, Max.Score: 1970.49, Time: 12:13:54 \n",
      "Ep. 2890, Timestep 1256371,  Ep.Timesteps 1000, Score: 1779.85, Avg.Score: 1546.06, Max.Score: 1970.49, Time: 12:19:05 \n",
      "Ep. 2900, Timestep 1266371,  Ep.Timesteps 1000, Score: 1659.49, Avg.Score: 1629.76, Max.Score: 1970.49, Time: 12:24:56 \n",
      "Ep. 2910, Timestep 1275393,  Ep.Timesteps 1000, Score: 1806.20, Avg.Score: 1642.06, Max.Score: 1970.49, Time: 12:30:04 \n",
      "Ep. 2920, Timestep 1284517,  Ep.Timesteps 1000, Score: 1728.41, Avg.Score: 1635.41, Max.Score: 1970.49, Time: 12:35:21 \n",
      "Ep. 2930, Timestep 1292752,  Ep.Timesteps 1000, Score: 1788.37, Avg.Score: 1608.88, Max.Score: 1970.49, Time: 12:40:03 \n",
      "Ep. 2940, Timestep 1300355,  Ep.Timesteps 1000, Score: 1869.57, Avg.Score: 1591.93, Max.Score: 1970.49, Time: 12:44:23 \n",
      "Ep. 2950, Timestep 1309468,  Ep.Timesteps 1000, Score: 1943.83, Avg.Score: 1574.99, Max.Score: 1970.49, Time: 12:49:50 \n",
      "Ep. 2960, Timestep 1315588,  Ep.Timesteps 1000, Score: 1943.50, Avg.Score: 1518.95, Max.Score: 1970.49, Time: 12:53:21 \n",
      "Ep. 2970, Timestep 1325588,  Ep.Timesteps 1000, Score: 1910.89, Avg.Score: 1537.21, Max.Score: 1993.76, Time: 12:59:04 \n",
      "Ep. 2980, Timestep 1332334,  Ep.Timesteps 1000, Score: 1868.44, Avg.Score: 1553.48, Max.Score: 2006.16, Time: 13:03:00 \n",
      "Ep. 2990, Timestep 1341385,  Ep.Timesteps 1000, Score: 1777.64, Avg.Score: 1552.67, Max.Score: 2006.16, Time: 13:09:10 \n",
      "Ep. 3000, Timestep 1347529,  Ep.Timesteps 1000, Score: 1776.68, Avg.Score: 1485.98, Max.Score: 2006.16, Time: 13:12:59 \n",
      "Ep. 3010, Timestep 1357529,  Ep.Timesteps 1000, Score: 1907.01, Avg.Score: 1510.36, Max.Score: 2006.16, Time: 13:19:10 \n",
      "Ep. 3020, Timestep 1364967,  Ep.Timesteps 64, Score: 82.08, Avg.Score: 1484.55, Max.Score: 2006.16, Time: 13:24:23 \n",
      "Ep. 3030, Timestep 1374967,  Ep.Timesteps 1000, Score: 1868.93, Avg.Score: 1517.67, Max.Score: 2006.16, Time: 13:30:01 \n",
      "Ep. 3040, Timestep 1384967,  Ep.Timesteps 1000, Score: 1884.03, Avg.Score: 1565.66, Max.Score: 2006.16, Time: 13:36:16 \n",
      "Ep. 3050, Timestep 1394233,  Ep.Timesteps 266, Score: 493.25, Avg.Score: 1582.60, Max.Score: 2006.16, Time: 13:42:19 \n",
      "Ep. 3060, Timestep 1404233,  Ep.Timesteps 1000, Score: 1891.44, Avg.Score: 1655.85, Max.Score: 2006.16, Time: 13:47:58 \n",
      "Ep. 3070, Timestep 1410392,  Ep.Timesteps 1000, Score: 1992.18, Avg.Score: 1577.04, Max.Score: 2006.16, Time: 13:51:47 \n",
      "Ep. 3080, Timestep 1420392,  Ep.Timesteps 1000, Score: 1899.90, Avg.Score: 1634.68, Max.Score: 2005.43, Time: 13:57:49 \n",
      "Ep. 3090, Timestep 1427881,  Ep.Timesteps 275, Score: 456.77, Avg.Score: 1612.86, Max.Score: 2005.43, Time: 14:02:46 \n",
      "Ep. 3100, Timestep 1428726,  Ep.Timesteps 36, Score: 41.26, Avg.Score: 1515.37, Max.Score: 2005.43, Time: 14:03:25 \n",
      "Ep. 3110, Timestep 1430394,  Ep.Timesteps 1000, Score: 1625.57, Avg.Score: 1356.61, Max.Score: 2005.43, Time: 14:03:53 \n",
      "Ep. 3120, Timestep 1436756,  Ep.Timesteps 1000, Score: 1743.67, Avg.Score: 1330.94, Max.Score: 2005.43, Time: 14:07:47 \n",
      "Ep. 3130, Timestep 1441522,  Ep.Timesteps 1000, Score: 1694.08, Avg.Score: 1227.11, Max.Score: 2005.43, Time: 14:10:38 \n",
      "Ep. 3140, Timestep 1450572,  Ep.Timesteps 1000, Score: 1835.93, Avg.Score: 1201.61, Max.Score: 2005.43, Time: 14:16:01 \n",
      "Ep. 3150, Timestep 1460120,  Ep.Timesteps 548, Score: 1027.94, Avg.Score: 1199.57, Max.Score: 1993.20, Time: 14:22:20 \n",
      "Ep. 3160, Timestep 1469172,  Ep.Timesteps 52, Score: 82.73, Avg.Score: 1187.71, Max.Score: 2075.02, Time: 14:28:19 \n",
      "Ep. 3170, Timestep 1475772,  Ep.Timesteps 1000, Score: 1906.53, Avg.Score: 1199.90, Max.Score: 2075.02, Time: 14:32:14 \n",
      "Ep. 3180, Timestep 1485772,  Ep.Timesteps 1000, Score: 2021.70, Avg.Score: 1206.39, Max.Score: 2075.02, Time: 14:38:41 \n",
      "Ep. 3190, Timestep 1493978,  Ep.Timesteps 21, Score: 26.73, Avg.Score: 1224.08, Max.Score: 2075.02, Time: 14:44:33 \n",
      "Ep. 3200, Timestep 1502143,  Ep.Timesteps 1000, Score: 1856.22, Avg.Score: 1371.81, Max.Score: 2075.02, Time: 14:49:12 \n",
      "Ep. 3210, Timestep 1512143,  Ep.Timesteps 1000, Score: 1933.77, Avg.Score: 1538.83, Max.Score: 2075.02, Time: 14:55:37 \n",
      "Ep. 3220, Timestep 1521323,  Ep.Timesteps 1000, Score: 1951.09, Avg.Score: 1606.49, Max.Score: 2075.02, Time: 15:01:32 \n",
      "Ep. 3230, Timestep 1531323,  Ep.Timesteps 1000, Score: 1938.48, Avg.Score: 1712.28, Max.Score: 2075.02, Time: 15:07:59 \n",
      "Ep. 3240, Timestep 1541323,  Ep.Timesteps 1000, Score: 1811.11, Avg.Score: 1733.52, Max.Score: 2075.02, Time: 15:14:25 \n",
      "Ep. 3250, Timestep 1551323,  Ep.Timesteps 1000, Score: 1849.01, Avg.Score: 1741.49, Max.Score: 2075.02, Time: 15:20:52 \n",
      "Ep. 3260, Timestep 1560507,  Ep.Timesteps 1000, Score: 2000.95, Avg.Score: 1740.80, Max.Score: 2030.79, Time: 15:26:47 \n",
      "Ep. 3270, Timestep 1570436,  Ep.Timesteps 1000, Score: 1910.66, Avg.Score: 1803.58, Max.Score: 2030.79, Time: 15:33:05 \n",
      "Ep. 3280, Timestep 1578704,  Ep.Timesteps 172, Score: 315.72, Avg.Score: 1771.49, Max.Score: 2032.84, Time: 15:38:53 \n",
      "Ep. 3290, Timestep 1583412,  Ep.Timesteps 232, Score: 400.71, Avg.Score: 1700.12, Max.Score: 2032.84, Time: 15:42:07 \n",
      "Ep. 3300, Timestep 1592212,  Ep.Timesteps 374, Score: 715.88, Avg.Score: 1708.91, Max.Score: 2032.84, Time: 15:47:38 \n",
      "Ep. 3310, Timestep 1602212,  Ep.Timesteps 1000, Score: 2009.94, Avg.Score: 1712.08, Max.Score: 2032.84, Time: 15:53:27 \n",
      "Ep. 3320, Timestep 1610857,  Ep.Timesteps 1000, Score: 1925.80, Avg.Score: 1699.17, Max.Score: 2032.84, Time: 15:58:25 \n",
      "Ep. 3330, Timestep 1617135,  Ep.Timesteps 110, Score: 181.93, Avg.Score: 1631.12, Max.Score: 2032.84, Time: 16:02:30 \n",
      "Ep. 3340, Timestep 1626366,  Ep.Timesteps 1000, Score: 1979.85, Avg.Score: 1626.51, Max.Score: 2047.46, Time: 16:07:26 \n",
      "Ep. 3350, Timestep 1635535,  Ep.Timesteps 1000, Score: 1917.49, Avg.Score: 1619.53, Max.Score: 2047.46, Time: 16:12:40 \n",
      "Ep. 3360, Timestep 1644767,  Ep.Timesteps 1000, Score: 2100.59, Avg.Score: 1627.58, Max.Score: 2100.59, Time: 16:17:59 \n",
      "Ep. 3370, Timestep 1653471,  Ep.Timesteps 205, Score: 368.43, Avg.Score: 1609.10, Max.Score: 2100.59, Time: 16:23:29 \n",
      "Ep. 3380, Timestep 1663471,  Ep.Timesteps 1000, Score: 2031.66, Avg.Score: 1648.23, Max.Score: 2100.59, Time: 16:28:49 \n",
      "Ep. 3390, Timestep 1673471,  Ep.Timesteps 1000, Score: 1957.82, Avg.Score: 1756.93, Max.Score: 2100.59, Time: 16:34:38 \n",
      "Ep. 3400, Timestep 1681460,  Ep.Timesteps 1000, Score: 1975.81, Avg.Score: 1744.42, Max.Score: 2100.59, Time: 16:39:13 \n",
      "Ep. 3410, Timestep 1689683,  Ep.Timesteps 29, Score: 34.93, Avg.Score: 1715.79, Max.Score: 2119.28, Time: 16:44:25 \n",
      "Ep. 3420, Timestep 1699683,  Ep.Timesteps 1000, Score: 1895.88, Avg.Score: 1747.25, Max.Score: 2119.28, Time: 16:49:46 \n",
      "Ep. 3430, Timestep 1707417,  Ep.Timesteps 1000, Score: 2035.19, Avg.Score: 1782.77, Max.Score: 2119.28, Time: 16:54:11 \n",
      "Ep. 3440, Timestep 1717417,  Ep.Timesteps 1000, Score: 2012.74, Avg.Score: 1802.41, Max.Score: 2119.28, Time: 16:59:56 \n",
      "Ep. 3450, Timestep 1727417,  Ep.Timesteps 1000, Score: 1966.08, Avg.Score: 1823.77, Max.Score: 2119.28, Time: 17:05:53 \n",
      "Ep. 3460, Timestep 1737417,  Ep.Timesteps 1000, Score: 1974.15, Avg.Score: 1837.80, Max.Score: 2119.28, Time: 17:11:44 \n",
      "Ep. 3470, Timestep 1746444,  Ep.Timesteps 1000, Score: 1927.84, Avg.Score: 1846.41, Max.Score: 2119.28, Time: 17:16:57 \n",
      "Ep. 3480, Timestep 1756444,  Ep.Timesteps 1000, Score: 1917.59, Avg.Score: 1839.09, Max.Score: 2119.28, Time: 17:22:50 \n",
      "Ep. 3490, Timestep 1763230,  Ep.Timesteps 18, Score: 18.58, Avg.Score: 1773.96, Max.Score: 2119.28, Time: 17:27:15 \n",
      "Ep. 3500, Timestep 1769236,  Ep.Timesteps 970, Score: 1886.70, Avg.Score: 1733.38, Max.Score: 2119.28, Time: 17:30:13 \n",
      "Ep. 3510, Timestep 1778304,  Ep.Timesteps 1000, Score: 1781.52, Avg.Score: 1743.80, Max.Score: 2080.34, Time: 17:35:54 \n",
      "Ep. 3520, Timestep 1787863,  Ep.Timesteps 1000, Score: 1857.74, Avg.Score: 1726.95, Max.Score: 2074.41, Time: 17:42:03 \n",
      "Ep. 3530, Timestep 1797143,  Ep.Timesteps 1000, Score: 1746.25, Avg.Score: 1750.04, Max.Score: 2074.41, Time: 17:47:59 \n",
      "Ep. 3540, Timestep 1807143,  Ep.Timesteps 1000, Score: 1910.22, Avg.Score: 1738.55, Max.Score: 2074.41, Time: 17:54:29 \n",
      "Ep. 3550, Timestep 1817143,  Ep.Timesteps 1000, Score: 1935.38, Avg.Score: 1729.79, Max.Score: 2062.30, Time: 18:01:05 \n",
      "Ep. 3560, Timestep 1823837,  Ep.Timesteps 1000, Score: 1397.42, Avg.Score: 1640.33, Max.Score: 2062.05, Time: 18:05:37 \n",
      "Ep. 3570, Timestep 1832903,  Ep.Timesteps 1000, Score: 1822.14, Avg.Score: 1604.46, Max.Score: 2062.05, Time: 18:11:30 \n",
      "Ep. 3580, Timestep 1837479,  Ep.Timesteps 123, Score: 167.38, Avg.Score: 1476.05, Max.Score: 2062.05, Time: 18:15:13 \n",
      "Ep. 3590, Timestep 1846836,  Ep.Timesteps 1000, Score: 1928.12, Avg.Score: 1522.28, Max.Score: 2039.45, Time: 18:20:56 \n",
      "Ep. 3600, Timestep 1853898,  Ep.Timesteps 15, Score: 12.66, Avg.Score: 1540.00, Max.Score: 2039.45, Time: 18:25:45 \n",
      "Ep. 3610, Timestep 1859044,  Ep.Timesteps 1000, Score: 1973.39, Avg.Score: 1461.81, Max.Score: 2039.45, Time: 18:28:18 \n",
      "Ep. 3620, Timestep 1868176,  Ep.Timesteps 1000, Score: 1918.00, Avg.Score: 1456.93, Max.Score: 2039.45, Time: 18:33:51 \n",
      "Ep. 3630, Timestep 1878176,  Ep.Timesteps 1000, Score: 2007.71, Avg.Score: 1472.72, Max.Score: 2039.45, Time: 18:39:56 \n",
      "Ep. 3640, Timestep 1887200,  Ep.Timesteps 1000, Score: 1945.87, Avg.Score: 1465.51, Max.Score: 2067.14, Time: 18:45:49 \n",
      "Ep. 3650, Timestep 1896232,  Ep.Timesteps 1000, Score: 2049.26, Avg.Score: 1453.70, Max.Score: 2067.14, Time: 18:51:51 \n",
      "Ep. 3660, Timestep 1906232,  Ep.Timesteps 1000, Score: 1872.63, Avg.Score: 1543.48, Max.Score: 2067.14, Time: 18:58:37 \n",
      "Ep. 3670, Timestep 1916232,  Ep.Timesteps 1000, Score: 1948.12, Avg.Score: 1596.09, Max.Score: 2067.14, Time: 19:04:59 \n",
      "Ep. 3680, Timestep 1926232,  Ep.Timesteps 1000, Score: 1896.91, Avg.Score: 1724.02, Max.Score: 2067.14, Time: 19:11:49 \n",
      "Ep. 3690, Timestep 1936232,  Ep.Timesteps 1000, Score: 2025.62, Avg.Score: 1741.66, Max.Score: 2067.14, Time: 19:18:35 \n",
      "Ep. 3700, Timestep 1946232,  Ep.Timesteps 1000, Score: 2002.00, Avg.Score: 1808.68, Max.Score: 2093.33, Time: 19:24:44 \n",
      "Ep. 3710, Timestep 1955694,  Ep.Timesteps 1000, Score: 1825.43, Avg.Score: 1882.63, Max.Score: 2093.33, Time: 19:30:20 \n",
      "Ep. 3720, Timestep 1965149,  Ep.Timesteps 1000, Score: 2069.15, Avg.Score: 1893.23, Max.Score: 2093.33, Time: 19:36:16 \n",
      "Ep. 3730, Timestep 1974012,  Ep.Timesteps 1000, Score: 2024.49, Avg.Score: 1879.53, Max.Score: 2093.33, Time: 19:41:45 \n",
      "Ep. 3740, Timestep 1983044,  Ep.Timesteps 1000, Score: 2020.34, Avg.Score: 1881.36, Max.Score: 2093.33, Time: 19:47:28 \n",
      "Ep. 3750, Timestep 1991278,  Ep.Timesteps 1000, Score: 2004.21, Avg.Score: 1869.07, Max.Score: 2117.69, Time: 19:52:39 \n",
      "Ep. 3760, Timestep 2000665,  Ep.Timesteps 1000, Score: 2019.62, Avg.Score: 1856.94, Max.Score: 2117.69, Time: 19:58:51 \n",
      "Ep. 3770, Timestep 2008312,  Ep.Timesteps 1000, Score: 2061.19, Avg.Score: 1815.48, Max.Score: 2117.69, Time: 20:03:24 \n",
      "Ep. 3780, Timestep 2012415,  Ep.Timesteps 732, Score: 1473.30, Avg.Score: 1703.09, Max.Score: 2117.69, Time: 20:06:28 \n",
      "Ep. 3790, Timestep 2022415,  Ep.Timesteps 1000, Score: 2048.15, Avg.Score: 1705.69, Max.Score: 2117.69, Time: 20:12:36 \n",
      "Ep. 3800, Timestep 2031492,  Ep.Timesteps 77, Score: 105.22, Avg.Score: 1687.93, Max.Score: 2117.69, Time: 20:18:22 \n",
      "Ep. 3810, Timestep 2041492,  Ep.Timesteps 1000, Score: 2054.04, Avg.Score: 1716.76, Max.Score: 2117.69, Time: 20:24:00 \n",
      "Ep. 3820, Timestep 2050648,  Ep.Timesteps 156, Score: 295.34, Avg.Score: 1721.20, Max.Score: 2117.69, Time: 20:30:34 \n",
      "Ep. 3830, Timestep 2059764,  Ep.Timesteps 1000, Score: 2079.05, Avg.Score: 1730.91, Max.Score: 2137.50, Time: 20:35:40 \n",
      "Ep. 3840, Timestep 2067382,  Ep.Timesteps 165, Score: 273.22, Avg.Score: 1705.56, Max.Score: 2137.50, Time: 20:40:58 \n",
      "Ep. 3850, Timestep 2075737,  Ep.Timesteps 1000, Score: 2088.79, Avg.Score: 1710.58, Max.Score: 2143.87, Time: 20:45:30 \n",
      "Ep. 3860, Timestep 2084440,  Ep.Timesteps 1000, Score: 2109.77, Avg.Score: 1702.86, Max.Score: 2143.87, Time: 20:50:44 \n",
      "Ep. 3870, Timestep 2090522,  Ep.Timesteps 40, Score: 57.42, Avg.Score: 1669.89, Max.Score: 2143.87, Time: 20:54:46 \n",
      "Ep. 3880, Timestep 2100522,  Ep.Timesteps 1000, Score: 2040.61, Avg.Score: 1789.60, Max.Score: 2143.87, Time: 21:00:03 \n",
      "Ep. 3890, Timestep 2110522,  Ep.Timesteps 1000, Score: 1896.72, Avg.Score: 1786.28, Max.Score: 2143.87, Time: 21:05:49 \n",
      "Ep. 3900, Timestep 2119599,  Ep.Timesteps 1000, Score: 1975.91, Avg.Score: 1783.13, Max.Score: 2143.87, Time: 21:11:05 \n",
      "Ep. 3910, Timestep 2129599,  Ep.Timesteps 1000, Score: 2100.29, Avg.Score: 1782.68, Max.Score: 2143.87, Time: 21:16:49 \n",
      "Ep. 3920, Timestep 2138033,  Ep.Timesteps 111, Score: 202.26, Avg.Score: 1764.04, Max.Score: 2143.87, Time: 21:22:09 \n",
      "Ep. 3930, Timestep 2141131,  Ep.Timesteps 31, Score: 31.37, Avg.Score: 1635.42, Max.Score: 2143.87, Time: 21:23:57 \n",
      "Ep. 3940, Timestep 2149847,  Ep.Timesteps 15, Score: 16.76, Avg.Score: 1654.87, Max.Score: 2143.87, Time: 21:29:00 \n",
      "Ep. 3950, Timestep 2158700,  Ep.Timesteps 431, Score: 826.83, Avg.Score: 1661.06, Max.Score: 2117.05, Time: 21:33:53 \n",
      "Ep. 3960, Timestep 2160067,  Ep.Timesteps 107, Score: 193.69, Avg.Score: 1507.56, Max.Score: 2114.25, Time: 21:34:53 \n",
      "Ep. 3970, Timestep 2166619,  Ep.Timesteps 1000, Score: 2120.41, Avg.Score: 1517.95, Max.Score: 2120.41, Time: 21:38:12 \n",
      "Ep. 3980, Timestep 2175698,  Ep.Timesteps 1000, Score: 2047.27, Avg.Score: 1500.06, Max.Score: 2120.41, Time: 21:43:27 \n",
      "Ep. 3990, Timestep 2185698,  Ep.Timesteps 1000, Score: 2048.02, Avg.Score: 1503.86, Max.Score: 2120.41, Time: 21:49:09 \n",
      "Ep. 4000, Timestep 2194860,  Ep.Timesteps 1000, Score: 1898.76, Avg.Score: 1510.40, Max.Score: 2120.41, Time: 21:54:26 \n",
      "Ep. 4010, Timestep 2201777,  Ep.Timesteps 768, Score: 1515.13, Avg.Score: 1449.82, Max.Score: 2136.93, Time: 21:58:32 \n",
      "Ep. 4020, Timestep 2211777,  Ep.Timesteps 1000, Score: 2108.88, Avg.Score: 1486.56, Max.Score: 2169.88, Time: 22:04:07 \n",
      "Ep. 4030, Timestep 2218765,  Ep.Timesteps 1000, Score: 2018.52, Avg.Score: 1569.23, Max.Score: 2188.58, Time: 22:08:08 \n",
      "Ep. 4040, Timestep 2228765,  Ep.Timesteps 1000, Score: 2049.51, Avg.Score: 1596.17, Max.Score: 2188.58, Time: 22:13:52 \n",
      "Ep. 4050, Timestep 2235345,  Ep.Timesteps 1000, Score: 1836.95, Avg.Score: 1538.55, Max.Score: 2188.58, Time: 22:17:38 \n",
      "Ep. 4060, Timestep 2245274,  Ep.Timesteps 1000, Score: 2017.86, Avg.Score: 1708.31, Max.Score: 2188.58, Time: 22:23:21 \n",
      "Ep. 4070, Timestep 2254348,  Ep.Timesteps 1000, Score: 1957.66, Avg.Score: 1758.09, Max.Score: 2188.58, Time: 22:28:34 \n",
      "Ep. 4080, Timestep 2264022,  Ep.Timesteps 674, Score: 1304.17, Avg.Score: 1771.42, Max.Score: 2188.58, Time: 22:34:15 \n",
      "Ep. 4090, Timestep 2273472,  Ep.Timesteps 1000, Score: 2061.61, Avg.Score: 1759.76, Max.Score: 2188.58, Time: 22:39:33 \n",
      "Ep. 4100, Timestep 2276876,  Ep.Timesteps 58, Score: 59.57, Avg.Score: 1638.58, Max.Score: 2188.58, Time: 22:42:02 \n",
      "Ep. 4110, Timestep 2281277,  Ep.Timesteps 53, Score: 58.64, Avg.Score: 1580.96, Max.Score: 2188.58, Time: 22:44:33 \n",
      "Ep. 4120, Timestep 2286553,  Ep.Timesteps 1000, Score: 2058.73, Avg.Score: 1478.81, Max.Score: 2188.58, Time: 22:47:04 \n",
      "Ep. 4130, Timestep 2293684,  Ep.Timesteps 446, Score: 892.02, Avg.Score: 1477.42, Max.Score: 2180.16, Time: 22:51:27 \n",
      "Ep. 4140, Timestep 2299388,  Ep.Timesteps 779, Score: 1595.78, Avg.Score: 1388.87, Max.Score: 2180.16, Time: 22:54:37 \n",
      "Ep. 4150, Timestep 2307594,  Ep.Timesteps 1000, Score: 2063.42, Avg.Score: 1436.17, Max.Score: 2180.16, Time: 23:00:11 \n",
      "Ep. 4160, Timestep 2317594,  Ep.Timesteps 1000, Score: 2070.48, Avg.Score: 1451.86, Max.Score: 2185.82, Time: 23:06:29 \n",
      "Ep. 4170, Timestep 2326682,  Ep.Timesteps 1000, Score: 2045.26, Avg.Score: 1458.59, Max.Score: 2185.82, Time: 23:12:29 \n",
      "Ep. 4180, Timestep 2334249,  Ep.Timesteps 1000, Score: 1997.43, Avg.Score: 1418.34, Max.Score: 2185.82, Time: 23:17:17 \n",
      "Ep. 4190, Timestep 2339096,  Ep.Timesteps 1000, Score: 2159.32, Avg.Score: 1328.72, Max.Score: 2185.82, Time: 23:20:24 \n",
      "Ep. 4200, Timestep 2347465,  Ep.Timesteps 1000, Score: 2004.61, Avg.Score: 1435.14, Max.Score: 2185.82, Time: 23:25:52 \n",
      "Ep. 4210, Timestep 2357465,  Ep.Timesteps 1000, Score: 2134.82, Avg.Score: 1563.22, Max.Score: 2187.28, Time: 23:32:04 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 4220, Timestep 2366217,  Ep.Timesteps 1000, Score: 2083.86, Avg.Score: 1644.19, Max.Score: 2203.03, Time: 23:37:52 \n",
      "Ep. 4230, Timestep 2375281,  Ep.Timesteps 1000, Score: 2068.88, Avg.Score: 1692.35, Max.Score: 2203.03, Time: 23:43:37 \n",
      "Ep. 4240, Timestep 2385281,  Ep.Timesteps 1000, Score: 2016.66, Avg.Score: 1785.44, Max.Score: 2203.03, Time: 23:49:48 \n",
      "Ep. 4250, Timestep 2395281,  Ep.Timesteps 1000, Score: 2064.15, Avg.Score: 1822.12, Max.Score: 2203.03, Time: 23:55:50 \n",
      "Ep. 4260, Timestep 2404464,  Ep.Timesteps 1000, Score: 2141.68, Avg.Score: 1798.81, Max.Score: 2203.03, Time: 24:01:11 \n",
      "Ep. 4270, Timestep 2414224,  Ep.Timesteps 1000, Score: 2063.64, Avg.Score: 1812.46, Max.Score: 2203.03, Time: 24:06:51 \n",
      "Ep. 4280, Timestep 2424224,  Ep.Timesteps 1000, Score: 2158.47, Avg.Score: 1865.77, Max.Score: 2203.03, Time: 24:12:52 \n",
      "Ep. 4290, Timestep 2432914,  Ep.Timesteps 1000, Score: 2097.12, Avg.Score: 1943.38, Max.Score: 2203.03, Time: 24:17:54 \n",
      "Ep. 4300, Timestep 2442914,  Ep.Timesteps 1000, Score: 2124.97, Avg.Score: 1969.38, Max.Score: 2203.03, Time: 24:23:43 \n",
      "Ep. 4310, Timestep 2450353,  Ep.Timesteps 494, Score: 928.15, Avg.Score: 1908.08, Max.Score: 2203.03, Time: 24:28:20 \n",
      "Ep. 4320, Timestep 2457352,  Ep.Timesteps 241, Score: 416.85, Avg.Score: 1866.87, Max.Score: 2194.67, Time: 24:32:31 \n",
      "Ep. 4330, Timestep 2466810,  Ep.Timesteps 1000, Score: 2050.65, Avg.Score: 1877.75, Max.Score: 2194.67, Time: 24:37:41 \n",
      "Ep. 4340, Timestep 2474310,  Ep.Timesteps 1000, Score: 2034.82, Avg.Score: 1822.43, Max.Score: 2179.11, Time: 24:42:50 \n",
      "Ep. 4350, Timestep 2482578,  Ep.Timesteps 119, Score: 183.23, Avg.Score: 1773.14, Max.Score: 2179.11, Time: 24:49:05 \n",
      "Ep. 4360, Timestep 2485836,  Ep.Timesteps 1000, Score: 1928.64, Avg.Score: 1645.77, Max.Score: 2179.11, Time: 24:50:44 \n",
      "Ep. 4370, Timestep 2493643,  Ep.Timesteps 1000, Score: 1847.18, Avg.Score: 1595.04, Max.Score: 2179.11, Time: 24:56:01 \n",
      "Ep. 4380, Timestep 2503643,  Ep.Timesteps 1000, Score: 1816.89, Avg.Score: 1582.62, Max.Score: 2179.11, Time: 25:02:50 \n",
      "Ep. 4390, Timestep 2512801,  Ep.Timesteps 1000, Score: 1912.62, Avg.Score: 1575.96, Max.Score: 2179.11, Time: 25:08:34 \n",
      "Ep. 4400, Timestep 2521307,  Ep.Timesteps 1000, Score: 1957.61, Avg.Score: 1539.69, Max.Score: 2179.11, Time: 25:14:36 \n",
      "Ep. 4410, Timestep 2529733,  Ep.Timesteps 1000, Score: 2013.78, Avg.Score: 1557.85, Max.Score: 2179.11, Time: 25:19:56 \n",
      "Ep. 4420, Timestep 2537872,  Ep.Timesteps 1000, Score: 1932.29, Avg.Score: 1571.84, Max.Score: 2174.10, Time: 25:25:19 \n",
      "Ep. 4430, Timestep 2546939,  Ep.Timesteps 1000, Score: 2070.71, Avg.Score: 1559.99, Max.Score: 2138.28, Time: 25:30:48 \n",
      "Ep. 4440, Timestep 2556939,  Ep.Timesteps 1000, Score: 2120.03, Avg.Score: 1615.76, Max.Score: 2138.28, Time: 25:37:19 \n",
      "Ep. 4450, Timestep 2565484,  Ep.Timesteps 1000, Score: 2114.78, Avg.Score: 1636.27, Max.Score: 2149.98, Time: 25:42:37 \n",
      "Ep. 4460, Timestep 2574034,  Ep.Timesteps 272, Score: 517.80, Avg.Score: 1750.55, Max.Score: 2149.98, Time: 25:48:21 \n",
      "Ep. 4470, Timestep 2583336,  Ep.Timesteps 1000, Score: 2124.70, Avg.Score: 1785.31, Max.Score: 2149.98, Time: 25:53:36 \n",
      "Ep. 4480, Timestep 2593336,  Ep.Timesteps 1000, Score: 2141.57, Avg.Score: 1800.17, Max.Score: 2175.55, Time: 25:59:24 \n",
      "Ep. 4490, Timestep 2602587,  Ep.Timesteps 1000, Score: 2120.32, Avg.Score: 1827.26, Max.Score: 2187.10, Time: 26:05:06 \n",
      "Ep. 4500, Timestep 2611687,  Ep.Timesteps 1000, Score: 2285.92, Avg.Score: 1862.79, Max.Score: 2285.92, Time: 26:10:22 \n",
      "Ep. 4510, Timestep 2621687,  Ep.Timesteps 1000, Score: 2175.41, Avg.Score: 1910.66, Max.Score: 2285.92, Time: 26:16:08 \n",
      "Ep. 4520, Timestep 2631344,  Ep.Timesteps 1000, Score: 2132.51, Avg.Score: 1957.53, Max.Score: 2285.92, Time: 26:21:45 \n",
      "Ep. 4530, Timestep 2640459,  Ep.Timesteps 1000, Score: 2114.00, Avg.Score: 1962.02, Max.Score: 2285.92, Time: 26:27:02 \n",
      "Ep. 4540, Timestep 2649506,  Ep.Timesteps 1000, Score: 2125.41, Avg.Score: 1949.53, Max.Score: 2285.92, Time: 26:32:13 \n",
      "Ep. 4550, Timestep 2658375,  Ep.Timesteps 1000, Score: 2087.71, Avg.Score: 1965.82, Max.Score: 2285.92, Time: 26:37:24 \n",
      "Ep. 4560, Timestep 2667821,  Ep.Timesteps 446, Score: 931.63, Avg.Score: 1999.10, Max.Score: 2285.92, Time: 26:43:09 \n",
      "Ep. 4570, Timestep 2677821,  Ep.Timesteps 1000, Score: 2114.71, Avg.Score: 2029.42, Max.Score: 2285.92, Time: 26:48:36 \n",
      "Ep. 4580, Timestep 2687821,  Ep.Timesteps 1000, Score: 2152.60, Avg.Score: 2030.50, Max.Score: 2285.92, Time: 26:54:27 \n",
      "Ep. 4590, Timestep 2693539,  Ep.Timesteps 86, Score: 110.51, Avg.Score: 1953.81, Max.Score: 2285.92, Time: 26:58:15 \n",
      "Ep. 4600, Timestep 2694184,  Ep.Timesteps 101, Score: 132.58, Avg.Score: 1765.38, Max.Score: 2247.66, Time: 26:58:38 \n",
      "Ep. 4610, Timestep 2697886,  Ep.Timesteps 885, Score: 1600.27, Avg.Score: 1613.12, Max.Score: 2247.66, Time: 27:00:23 \n",
      "Ep. 4620, Timestep 2702161,  Ep.Timesteps 84, Score: 75.17, Avg.Score: 1483.25, Max.Score: 2247.66, Time: 27:03:15 \n",
      "Ep. 4630, Timestep 2703670,  Ep.Timesteps 899, Score: 1542.08, Avg.Score: 1311.37, Max.Score: 2247.66, Time: 27:03:41 \n",
      "Ep. 4640, Timestep 2710984,  Ep.Timesteps 1000, Score: 1779.11, Avg.Score: 1244.68, Max.Score: 2247.66, Time: 27:07:56 \n",
      "Ep. 4650, Timestep 2720984,  Ep.Timesteps 1000, Score: 2059.27, Avg.Score: 1236.89, Max.Score: 2247.66, Time: 27:13:44 \n",
      "Ep. 4660, Timestep 2730984,  Ep.Timesteps 1000, Score: 2074.29, Avg.Score: 1232.09, Max.Score: 2227.03, Time: 27:19:27 \n",
      "Ep. 4670, Timestep 2740984,  Ep.Timesteps 1000, Score: 2145.62, Avg.Score: 1225.38, Max.Score: 2190.66, Time: 27:25:18 \n",
      "Ep. 4680, Timestep 2750984,  Ep.Timesteps 1000, Score: 2027.77, Avg.Score: 1217.89, Max.Score: 2183.11, Time: 27:31:04 \n",
      "Ep. 4690, Timestep 2760984,  Ep.Timesteps 1000, Score: 2118.06, Avg.Score: 1307.42, Max.Score: 2183.11, Time: 27:36:52 \n",
      "Ep. 4700, Timestep 2770984,  Ep.Timesteps 1000, Score: 2080.26, Avg.Score: 1511.53, Max.Score: 2202.96, Time: 27:42:39 \n",
      "Ep. 4710, Timestep 2780984,  Ep.Timesteps 1000, Score: 2146.87, Avg.Score: 1660.83, Max.Score: 2237.78, Time: 27:48:25 \n",
      "Ep. 4720, Timestep 2790984,  Ep.Timesteps 1000, Score: 2010.13, Avg.Score: 1795.33, Max.Score: 2237.78, Time: 27:54:15 \n",
      "Ep. 4730, Timestep 2800374,  Ep.Timesteps 1000, Score: 1967.58, Avg.Score: 1967.15, Max.Score: 2237.78, Time: 27:59:41 \n",
      "Ep. 4740, Timestep 2810374,  Ep.Timesteps 1000, Score: 2135.82, Avg.Score: 2050.29, Max.Score: 2237.78, Time: 28:05:29 \n",
      "Ep. 4750, Timestep 2819726,  Ep.Timesteps 1000, Score: 2116.73, Avg.Score: 2064.01, Max.Score: 2237.78, Time: 28:10:52 \n",
      "Ep. 4760, Timestep 2829375,  Ep.Timesteps 1000, Score: 2160.04, Avg.Score: 2071.19, Max.Score: 2237.78, Time: 28:16:28 \n",
      "Ep. 4770, Timestep 2839375,  Ep.Timesteps 1000, Score: 2211.58, Avg.Score: 2076.97, Max.Score: 2237.78, Time: 28:22:17 \n",
      "Ep. 4780, Timestep 2849375,  Ep.Timesteps 1000, Score: 2211.03, Avg.Score: 2090.85, Max.Score: 2290.49, Time: 28:28:10 \n",
      "Ep. 4790, Timestep 2859375,  Ep.Timesteps 1000, Score: 2229.42, Avg.Score: 2102.73, Max.Score: 2290.49, Time: 28:33:56 \n",
      "Ep. 4800, Timestep 2869375,  Ep.Timesteps 1000, Score: 2098.42, Avg.Score: 2108.00, Max.Score: 2290.49, Time: 28:39:46 \n",
      "Ep. 4810, Timestep 2879375,  Ep.Timesteps 1000, Score: 2171.75, Avg.Score: 2109.17, Max.Score: 2290.49, Time: 28:45:33 \n",
      "Ep. 4820, Timestep 2889375,  Ep.Timesteps 1000, Score: 2141.24, Avg.Score: 2117.72, Max.Score: 2290.49, Time: 28:51:22 \n",
      "Ep. 4830, Timestep 2898885,  Ep.Timesteps 1000, Score: 2063.11, Avg.Score: 2129.75, Max.Score: 2290.49, Time: 28:56:54 \n",
      "Ep. 4840, Timestep 2908580,  Ep.Timesteps 695, Score: 1394.35, Avg.Score: 2112.74, Max.Score: 2290.49, Time: 29:02:47 \n",
      "Ep. 4850, Timestep 2917615,  Ep.Timesteps 449, Score: 851.85, Avg.Score: 2106.46, Max.Score: 2290.49, Time: 29:08:28 \n",
      "Ep. 4860, Timestep 2927615,  Ep.Timesteps 1000, Score: 2139.86, Avg.Score: 2108.31, Max.Score: 2290.49, Time: 29:14:17 \n",
      "Ep. 4870, Timestep 2937615,  Ep.Timesteps 1000, Score: 2094.75, Avg.Score: 2103.81, Max.Score: 2290.49, Time: 29:20:19 \n",
      "Ep. 4880, Timestep 2945033,  Ep.Timesteps 1000, Score: 2003.67, Avg.Score: 2042.11, Max.Score: 2251.05, Time: 29:25:22 \n",
      "Ep. 4890, Timestep 2955033,  Ep.Timesteps 1000, Score: 2188.97, Avg.Score: 2040.24, Max.Score: 2272.05, Time: 29:31:54 \n",
      "Ep. 4900, Timestep 2962706,  Ep.Timesteps 1000, Score: 2085.20, Avg.Score: 1986.66, Max.Score: 2272.05, Time: 29:36:58 \n",
      "Ep. 4910, Timestep 2972706,  Ep.Timesteps 1000, Score: 2168.05, Avg.Score: 1988.53, Max.Score: 2272.05, Time: 29:43:31 \n",
      "Ep. 4920, Timestep 2982016,  Ep.Timesteps 1000, Score: 2142.83, Avg.Score: 1971.14, Max.Score: 2272.05, Time: 29:49:24 \n",
      "Ep. 4930, Timestep 2991521,  Ep.Timesteps 1000, Score: 2158.07, Avg.Score: 1972.70, Max.Score: 2272.05, Time: 29:56:21 \n",
      "Ep. 4940, Timestep 2999319,  Ep.Timesteps 1000, Score: 2200.11, Avg.Score: 1936.39, Max.Score: 2272.05, Time: 30:01:50 \n",
      "Ep. 4950, Timestep 3009319,  Ep.Timesteps 1000, Score: 2159.61, Avg.Score: 1959.32, Max.Score: 2272.05, Time: 30:08:06 \n",
      "Ep. 4960, Timestep 3019319,  Ep.Timesteps 1000, Score: 2231.54, Avg.Score: 1965.83, Max.Score: 2272.05, Time: 30:14:14 \n",
      "Ep. 4970, Timestep 3029319,  Ep.Timesteps 1000, Score: 2150.50, Avg.Score: 1974.39, Max.Score: 2316.80, Time: 30:20:03 \n",
      "Ep. 4980, Timestep 3039319,  Ep.Timesteps 1000, Score: 2241.90, Avg.Score: 2038.97, Max.Score: 2316.80, Time: 30:26:05 \n",
      "Ep. 4990, Timestep 3049319,  Ep.Timesteps 1000, Score: 2159.85, Avg.Score: 2039.44, Max.Score: 2316.80, Time: 30:32:06 \n",
      "Ep. 5000, Timestep 3059319,  Ep.Timesteps 1000, Score: 2158.56, Avg.Score: 2096.81, Max.Score: 2316.80, Time: 30:38:07 \n",
      "Ep. 5010, Timestep 3069319,  Ep.Timesteps 1000, Score: 2138.32, Avg.Score: 2095.86, Max.Score: 2316.80, Time: 30:44:38 \n",
      "Ep. 5020, Timestep 3079319,  Ep.Timesteps 1000, Score: 2232.55, Avg.Score: 2116.78, Max.Score: 2316.80, Time: 30:50:46 \n",
      "Ep. 5030, Timestep 3087693,  Ep.Timesteps 126, Score: 238.91, Avg.Score: 2096.56, Max.Score: 2316.80, Time: 30:56:39 \n",
      "Ep. 5040, Timestep 3096276,  Ep.Timesteps 1000, Score: 2180.19, Avg.Score: 2129.16, Max.Score: 2316.80, Time: 31:01:20 \n",
      "Ep. 5050, Timestep 3106276,  Ep.Timesteps 1000, Score: 2129.58, Avg.Score: 2132.14, Max.Score: 2316.80, Time: 31:07:38 \n",
      "Ep. 5060, Timestep 3116276,  Ep.Timesteps 1000, Score: 2150.53, Avg.Score: 2134.13, Max.Score: 2316.80, Time: 31:14:03 \n",
      "Ep. 5070, Timestep 3126213,  Ep.Timesteps 1000, Score: 2250.28, Avg.Score: 2133.18, Max.Score: 2327.45, Time: 31:20:55 \n",
      "Ep. 5080, Timestep 3136213,  Ep.Timesteps 1000, Score: 2207.17, Avg.Score: 2135.24, Max.Score: 2327.45, Time: 31:26:54 \n",
      "Ep. 5090, Timestep 3145259,  Ep.Timesteps 1000, Score: 2270.68, Avg.Score: 2119.89, Max.Score: 2327.45, Time: 31:32:18 \n",
      "Ep. 5100, Timestep 3155259,  Ep.Timesteps 1000, Score: 2234.72, Avg.Score: 2125.14, Max.Score: 2327.45, Time: 31:38:28 \n",
      "Ep. 5110, Timestep 3165259,  Ep.Timesteps 1000, Score: 2268.46, Avg.Score: 2134.50, Max.Score: 2327.45, Time: 31:44:30 \n",
      "Ep. 5120, Timestep 3175259,  Ep.Timesteps 1000, Score: 2190.38, Avg.Score: 2135.95, Max.Score: 2327.45, Time: 31:50:39 \n",
      "Ep. 5130, Timestep 3185259,  Ep.Timesteps 1000, Score: 2249.10, Avg.Score: 2173.70, Max.Score: 2327.45, Time: 31:56:49 \n",
      "Ep. 5140, Timestep 3194488,  Ep.Timesteps 1000, Score: 2270.34, Avg.Score: 2189.95, Max.Score: 2327.45, Time: 32:02:19 \n",
      "Ep. 5150, Timestep 3203816,  Ep.Timesteps 1000, Score: 2280.35, Avg.Score: 2181.60, Max.Score: 2327.45, Time: 32:08:10 \n",
      "Ep. 5160, Timestep 3213816,  Ep.Timesteps 1000, Score: 2303.42, Avg.Score: 2192.01, Max.Score: 2328.98, Time: 32:15:10 \n",
      "Ep. 5170, Timestep 3223064,  Ep.Timesteps 1000, Score: 2193.41, Avg.Score: 2181.13, Max.Score: 2328.98, Time: 32:21:05 \n",
      "Ep. 5180, Timestep 3233064,  Ep.Timesteps 1000, Score: 2267.07, Avg.Score: 2178.96, Max.Score: 2328.98, Time: 32:27:19 \n",
      "Ep. 5190, Timestep 3243064,  Ep.Timesteps 1000, Score: 2321.89, Avg.Score: 2196.90, Max.Score: 2328.98, Time: 32:33:47 \n",
      "Ep. 5200, Timestep 3253064,  Ep.Timesteps 1000, Score: 2288.92, Avg.Score: 2198.79, Max.Score: 2365.83, Time: 32:40:34 \n",
      "Ep. 5210, Timestep 3263064,  Ep.Timesteps 1000, Score: 2289.29, Avg.Score: 2200.95, Max.Score: 2365.83, Time: 32:47:31 \n",
      "Ep. 5220, Timestep 3271330,  Ep.Timesteps 1000, Score: 2262.85, Avg.Score: 2165.92, Max.Score: 2379.95, Time: 32:53:03 \n",
      "Ep. 5230, Timestep 3277878,  Ep.Timesteps 797, Score: 1737.86, Avg.Score: 2086.70, Max.Score: 2379.95, Time: 32:57:23 \n",
      "Ep. 5240, Timestep 3287878,  Ep.Timesteps 1000, Score: 2262.64, Avg.Score: 2102.91, Max.Score: 2379.95, Time: 33:02:58 \n",
      "Ep. 5250, Timestep 3297878,  Ep.Timesteps 1000, Score: 2259.15, Avg.Score: 2117.46, Max.Score: 2379.95, Time: 33:08:54 \n",
      "Ep. 5260, Timestep 3307878,  Ep.Timesteps 1000, Score: 2260.08, Avg.Score: 2115.42, Max.Score: 2379.95, Time: 33:14:35 \n",
      "Ep. 5270, Timestep 3317119,  Ep.Timesteps 241, Score: 479.67, Avg.Score: 2114.17, Max.Score: 2379.95, Time: 33:20:53 \n",
      "Ep. 5280, Timestep 3326977,  Ep.Timesteps 1000, Score: 2226.00, Avg.Score: 2117.72, Max.Score: 2379.95, Time: 33:26:07 \n",
      "Ep. 5290, Timestep 3336171,  Ep.Timesteps 194, Score: 391.29, Avg.Score: 2104.30, Max.Score: 2379.95, Time: 33:31:46 \n",
      "Ep. 5300, Timestep 3346171,  Ep.Timesteps 1000, Score: 2237.26, Avg.Score: 2102.27, Max.Score: 2379.95, Time: 33:37:03 \n",
      "Ep. 5310, Timestep 3355786,  Ep.Timesteps 1000, Score: 2251.15, Avg.Score: 2089.61, Max.Score: 2379.95, Time: 33:42:28 \n",
      "Ep. 5320, Timestep 3364890,  Ep.Timesteps 1000, Score: 2291.77, Avg.Score: 2107.91, Max.Score: 2354.41, Time: 33:47:38 \n",
      "Ep. 5330, Timestep 3374171,  Ep.Timesteps 1000, Score: 2335.95, Avg.Score: 2174.22, Max.Score: 2354.41, Time: 33:52:56 \n",
      "Ep. 5340, Timestep 3382415,  Ep.Timesteps 1000, Score: 2194.46, Avg.Score: 2140.83, Max.Score: 2354.41, Time: 33:57:36 \n",
      "Ep. 5350, Timestep 3390388,  Ep.Timesteps 1000, Score: 1985.49, Avg.Score: 2085.63, Max.Score: 2354.41, Time: 34:02:06 \n",
      "Ep. 5360, Timestep 3394265,  Ep.Timesteps 64, Score: 84.63, Avg.Score: 1926.22, Max.Score: 2354.41, Time: 34:04:48 \n",
      "Ep. 5370, Timestep 3395027,  Ep.Timesteps 91, Score: 107.88, Avg.Score: 1729.98, Max.Score: 2354.41, Time: 34:05:13 \n",
      "Ep. 5380, Timestep 3395744,  Ep.Timesteps 77, Score: 62.93, Avg.Score: 1513.66, Max.Score: 2354.41, Time: 34:05:40 \n",
      "Ep. 5390, Timestep 3398344,  Ep.Timesteps 68, Score: 83.69, Avg.Score: 1317.38, Max.Score: 2354.41, Time: 34:07:09 \n",
      "Ep. 5400, Timestep 3399222,  Ep.Timesteps 71, Score: 80.66, Avg.Score: 1100.07, Max.Score: 2354.41, Time: 34:07:39 \n",
      "Ep. 5410, Timestep 3399973,  Ep.Timesteps 94, Score: 115.93, Avg.Score: 894.24, Max.Score: 2354.41, Time: 34:08:04 \n",
      "Ep. 5420, Timestep 3400721,  Ep.Timesteps 42, Score: 32.84, Avg.Score: 695.33, Max.Score: 2351.06, Time: 34:08:31 \n",
      "Ep. 5430, Timestep 3401427,  Ep.Timesteps 86, Score: 94.67, Avg.Score: 491.97, Max.Score: 2351.06, Time: 34:08:54 \n",
      "Ep. 5440, Timestep 3402006,  Ep.Timesteps 73, Score: 67.20, Avg.Score: 307.59, Max.Score: 2265.56, Time: 34:09:14 \n",
      "Ep. 5450, Timestep 3402796,  Ep.Timesteps 61, Score: 45.90, Avg.Score: 146.91, Max.Score: 1949.85, Time: 34:09:41 \n",
      "Ep. 5460, Timestep 3403716,  Ep.Timesteps 69, Score: 76.21, Avg.Score: 89.58, Max.Score: 436.19, Time: 34:10:12 \n",
      "Ep. 5470, Timestep 3405787,  Ep.Timesteps 50, Score: 68.84, Avg.Score: 111.76, Max.Score: 1803.30, Time: 34:11:23 \n",
      "Ep. 5480, Timestep 3408451,  Ep.Timesteps 76, Score: 118.32, Avg.Score: 129.41, Max.Score: 1803.30, Time: 34:12:53 \n",
      "Ep. 5490, Timestep 3411402,  Ep.Timesteps 364, Score: 265.94, Avg.Score: 141.65, Max.Score: 1803.30, Time: 34:14:24 \n",
      "Ep. 5500, Timestep 3412510,  Ep.Timesteps 148, Score: 186.37, Avg.Score: 146.25, Max.Score: 1803.30, Time: 34:15:09 \n",
      "Ep. 5510, Timestep 3413970,  Ep.Timesteps 100, Score: 44.94, Avg.Score: 158.30, Max.Score: 1803.30, Time: 34:15:59 \n",
      "Ep. 5520, Timestep 3418767,  Ep.Timesteps 256, Score: 454.53, Avg.Score: 204.96, Max.Score: 1803.30, Time: 34:18:38 \n",
      "Ep. 5530, Timestep 3428476,  Ep.Timesteps 1000, Score: 1939.01, Avg.Score: 387.03, Max.Score: 2055.94, Time: 34:23:46 \n",
      "Ep. 5540, Timestep 3435092,  Ep.Timesteps 78, Score: 124.51, Avg.Score: 514.77, Max.Score: 2068.37, Time: 34:28:00 \n",
      "Ep. 5550, Timestep 3443813,  Ep.Timesteps 1000, Score: 2064.44, Avg.Score: 688.48, Max.Score: 2187.36, Time: 34:32:27 \n",
      "Ep. 5560, Timestep 3453813,  Ep.Timesteps 1000, Score: 2022.79, Avg.Score: 880.47, Max.Score: 2187.36, Time: 34:38:08 \n",
      "Ep. 5570, Timestep 3461883,  Ep.Timesteps 1000, Score: 2015.20, Avg.Score: 1013.06, Max.Score: 2238.71, Time: 34:42:41 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 5580, Timestep 3471883,  Ep.Timesteps 1000, Score: 2086.63, Avg.Score: 1195.04, Max.Score: 2238.71, Time: 34:48:20 \n",
      "Ep. 5590, Timestep 3481883,  Ep.Timesteps 1000, Score: 2058.47, Avg.Score: 1385.03, Max.Score: 2238.71, Time: 34:54:02 \n",
      "Ep. 5600, Timestep 3491169,  Ep.Timesteps 286, Score: 571.69, Avg.Score: 1571.59, Max.Score: 2238.71, Time: 34:59:39 \n",
      "Ep. 5610, Timestep 3500209,  Ep.Timesteps 1000, Score: 2117.53, Avg.Score: 1745.83, Max.Score: 2256.70, Time: 35:04:23 \n",
      "Ep. 5620, Timestep 3509671,  Ep.Timesteps 1000, Score: 2150.30, Avg.Score: 1899.26, Max.Score: 2269.35, Time: 35:09:46 \n",
      "Ep. 5630, Timestep 3519174,  Ep.Timesteps 1000, Score: 2167.54, Avg.Score: 1915.57, Max.Score: 2269.35, Time: 35:15:07 \n",
      "Ep. 5640, Timestep 3529174,  Ep.Timesteps 1000, Score: 2150.40, Avg.Score: 1996.39, Max.Score: 2269.35, Time: 35:20:48 \n",
      "Ep. 5650, Timestep 3536442,  Ep.Timesteps 118, Score: 208.21, Avg.Score: 1969.31, Max.Score: 2269.35, Time: 35:25:23 \n",
      "Ep. 5660, Timestep 3545073,  Ep.Timesteps 1000, Score: 2216.69, Avg.Score: 1948.35, Max.Score: 2269.35, Time: 35:29:47 \n",
      "Ep. 5670, Timestep 3555073,  Ep.Timesteps 1000, Score: 2071.84, Avg.Score: 1998.40, Max.Score: 2269.35, Time: 35:35:25 \n",
      "Ep. 5680, Timestep 3563197,  Ep.Timesteps 1000, Score: 2223.17, Avg.Score: 1958.21, Max.Score: 2269.35, Time: 35:40:03 \n",
      "Ep. 5690, Timestep 3572427,  Ep.Timesteps 1000, Score: 2167.13, Avg.Score: 1941.39, Max.Score: 2269.35, Time: 35:45:15 \n",
      "Ep. 5700, Timestep 3581721,  Ep.Timesteps 1000, Score: 2228.86, Avg.Score: 1945.38, Max.Score: 2269.35, Time: 35:50:31 \n",
      "Ep. 5710, Timestep 3590621,  Ep.Timesteps 928, Score: 2015.38, Avg.Score: 1949.92, Max.Score: 2315.48, Time: 35:55:38 \n",
      "Ep. 5720, Timestep 3595304,  Ep.Timesteps 227, Score: 436.09, Avg.Score: 1841.66, Max.Score: 2315.48, Time: 35:58:38 \n",
      "Ep. 5730, Timestep 3602805,  Ep.Timesteps 177, Score: 357.39, Avg.Score: 1803.50, Max.Score: 2315.48, Time: 36:02:53 \n",
      "Ep. 5740, Timestep 3612198,  Ep.Timesteps 1000, Score: 2244.36, Avg.Score: 1800.27, Max.Score: 2319.97, Time: 36:07:49 \n",
      "Ep. 5750, Timestep 3620348,  Ep.Timesteps 1000, Score: 2231.42, Avg.Score: 1824.04, Max.Score: 2319.97, Time: 36:12:25 \n",
      "Ep. 5760, Timestep 3630348,  Ep.Timesteps 1000, Score: 2143.57, Avg.Score: 1858.85, Max.Score: 2319.97, Time: 36:18:04 \n",
      "Ep. 5770, Timestep 3640348,  Ep.Timesteps 1000, Score: 2206.96, Avg.Score: 1860.24, Max.Score: 2319.97, Time: 36:23:45 \n",
      "Ep. 5780, Timestep 3650348,  Ep.Timesteps 1000, Score: 2197.15, Avg.Score: 1909.83, Max.Score: 2319.97, Time: 36:29:23 \n",
      "Ep. 5790, Timestep 3660348,  Ep.Timesteps 1000, Score: 2124.85, Avg.Score: 1922.35, Max.Score: 2319.97, Time: 36:35:01 \n",
      "Ep. 5800, Timestep 3668097,  Ep.Timesteps 1000, Score: 2151.04, Avg.Score: 1880.36, Max.Score: 2319.97, Time: 36:39:26 \n",
      "Ep. 5810, Timestep 3677193,  Ep.Timesteps 96, Score: 169.88, Avg.Score: 1871.96, Max.Score: 2319.97, Time: 36:45:03 \n",
      "Ep. 5820, Timestep 3686627,  Ep.Timesteps 1000, Score: 2153.67, Avg.Score: 1976.69, Max.Score: 2319.97, Time: 36:49:54 \n",
      "Ep. 5830, Timestep 3695825,  Ep.Timesteps 1000, Score: 2230.56, Avg.Score: 2004.07, Max.Score: 2319.97, Time: 36:55:08 \n",
      "Ep. 5840, Timestep 3705825,  Ep.Timesteps 1000, Score: 2238.75, Avg.Score: 2012.47, Max.Score: 2313.18, Time: 37:00:48 \n",
      "Ep. 5850, Timestep 3715825,  Ep.Timesteps 1000, Score: 2263.36, Avg.Score: 2055.77, Max.Score: 2280.12, Time: 37:06:29 \n",
      "Ep. 5860, Timestep 3723142,  Ep.Timesteps 1000, Score: 2195.42, Avg.Score: 1999.19, Max.Score: 2314.33, Time: 37:10:37 \n",
      "Ep. 5870, Timestep 3729364,  Ep.Timesteps 74, Score: -14.01, Avg.Score: 1915.05, Max.Score: 2314.33, Time: 37:14:37 \n",
      "Ep. 5880, Timestep 3732775,  Ep.Timesteps 51, Score: 37.54, Avg.Score: 1763.71, Max.Score: 2314.33, Time: 37:16:33 \n",
      "Ep. 5890, Timestep 3739813,  Ep.Timesteps 1000, Score: 2064.81, Avg.Score: 1696.40, Max.Score: 2314.33, Time: 37:20:03 \n",
      "Ep. 5900, Timestep 3744430,  Ep.Timesteps 1000, Score: 1654.01, Avg.Score: 1617.46, Max.Score: 2314.33, Time: 37:22:41 \n",
      "Ep. 5910, Timestep 3750565,  Ep.Timesteps 347, Score: 531.99, Avg.Score: 1529.31, Max.Score: 2314.33, Time: 37:26:30 \n",
      "Ep. 5920, Timestep 3756315,  Ep.Timesteps 87, Score: 128.88, Avg.Score: 1412.86, Max.Score: 2314.33, Time: 37:29:53 \n",
      "Ep. 5930, Timestep 3763030,  Ep.Timesteps 828, Score: 1516.01, Avg.Score: 1323.21, Max.Score: 2314.33, Time: 37:33:22 \n",
      "Ep. 5940, Timestep 3768124,  Ep.Timesteps 131, Score: 227.09, Avg.Score: 1201.10, Max.Score: 2314.33, Time: 37:36:38 \n",
      "Ep. 5950, Timestep 3770182,  Ep.Timesteps 164, Score: 286.40, Avg.Score: 1016.33, Max.Score: 2314.33, Time: 37:37:47 \n",
      "Ep. 5960, Timestep 3774968,  Ep.Timesteps 1000, Score: 2019.52, Avg.Score: 949.98, Max.Score: 2272.73, Time: 37:40:03 \n",
      "Ep. 5970, Timestep 3782561,  Ep.Timesteps 1000, Score: 2066.85, Avg.Score: 977.36, Max.Score: 2187.60, Time: 37:44:20 \n",
      "Ep. 5980, Timestep 3790673,  Ep.Timesteps 1000, Score: 2165.75, Avg.Score: 1079.29, Max.Score: 2187.60, Time: 37:48:54 \n",
      "Ep. 5990, Timestep 3799738,  Ep.Timesteps 1000, Score: 2216.28, Avg.Score: 1125.38, Max.Score: 2216.28, Time: 37:54:04 \n",
      "Ep. 6000, Timestep 3807907,  Ep.Timesteps 109, Score: 199.65, Avg.Score: 1219.54, Max.Score: 2245.98, Time: 37:59:07 \n",
      "Ep. 6010, Timestep 3817907,  Ep.Timesteps 1000, Score: 2221.55, Avg.Score: 1333.57, Max.Score: 2282.74, Time: 38:04:19 \n",
      "Ep. 6020, Timestep 3822299,  Ep.Timesteps 72, Score: 111.29, Avg.Score: 1336.04, Max.Score: 2282.74, Time: 38:07:20 \n",
      "Ep. 6030, Timestep 3822957,  Ep.Timesteps 56, Score: 74.64, Avg.Score: 1238.85, Max.Score: 2282.74, Time: 38:07:43 \n",
      "Ep. 6040, Timestep 3823557,  Ep.Timesteps 54, Score: 74.48, Avg.Score: 1150.13, Max.Score: 2282.74, Time: 38:08:03 \n",
      "Ep. 6050, Timestep 3824061,  Ep.Timesteps 72, Score: 102.53, Avg.Score: 1119.46, Max.Score: 2282.74, Time: 38:08:20 \n",
      "Ep. 6060, Timestep 3824812,  Ep.Timesteps 71, Score: 96.81, Avg.Score: 1036.28, Max.Score: 2282.74, Time: 38:08:46 \n",
      "Ep. 6070, Timestep 3825549,  Ep.Timesteps 56, Score: 62.90, Avg.Score: 887.17, Max.Score: 2282.74, Time: 38:09:11 \n",
      "Ep. 6080, Timestep 3826119,  Ep.Timesteps 79, Score: 121.08, Avg.Score: 726.92, Max.Score: 2282.74, Time: 38:09:30 \n",
      "Ep. 6090, Timestep 3826640,  Ep.Timesteps 78, Score: 122.25, Avg.Score: 544.61, Max.Score: 2282.74, Time: 38:09:48 \n",
      "Ep. 6100, Timestep 3827091,  Ep.Timesteps 39, Score: 34.36, Avg.Score: 371.99, Max.Score: 2282.74, Time: 38:10:05 \n",
      "Ep. 6110, Timestep 3827521,  Ep.Timesteps 45, Score: 47.28, Avg.Score: 159.09, Max.Score: 2255.71, Time: 38:10:19 \n",
      "Ep. 6120, Timestep 3828095,  Ep.Timesteps 55, Score: 66.00, Avg.Score: 75.33, Max.Score: 267.37, Time: 38:10:39 \n",
      "Ep. 6130, Timestep 3828936,  Ep.Timesteps 60, Score: 77.63, Avg.Score: 78.81, Max.Score: 298.70, Time: 38:11:07 \n",
      "Ep. 6140, Timestep 3830575,  Ep.Timesteps 78, Score: 61.64, Avg.Score: 96.04, Max.Score: 1766.14, Time: 38:12:02 \n",
      "Ep. 6150, Timestep 3831289,  Ep.Timesteps 54, Score: 30.01, Avg.Score: 97.44, Max.Score: 1766.14, Time: 38:12:27 \n",
      "Ep. 6160, Timestep 3832262,  Ep.Timesteps 73, Score: 50.23, Avg.Score: 99.27, Max.Score: 1766.14, Time: 38:13:00 \n",
      "Ep. 6170, Timestep 3833018,  Ep.Timesteps 89, Score: 126.05, Avg.Score: 96.97, Max.Score: 1766.14, Time: 38:13:25 \n",
      "Ep. 6180, Timestep 3835894,  Ep.Timesteps 185, Score: 300.55, Avg.Score: 136.18, Max.Score: 1795.82, Time: 38:15:00 \n",
      "Ep. 6190, Timestep 3839036,  Ep.Timesteps 78, Score: 61.71, Avg.Score: 171.40, Max.Score: 1914.05, Time: 38:16:50 \n",
      "Ep. 6200, Timestep 3849036,  Ep.Timesteps 1000, Score: 2119.63, Avg.Score: 361.62, Max.Score: 2119.63, Time: 38:22:03 \n",
      "Ep. 6210, Timestep 3858077,  Ep.Timesteps 1000, Score: 2121.51, Avg.Score: 547.24, Max.Score: 2197.62, Time: 38:27:10 \n",
      "Ep. 6220, Timestep 3868077,  Ep.Timesteps 1000, Score: 2030.81, Avg.Score: 747.23, Max.Score: 2197.62, Time: 38:32:49 \n",
      "Ep. 6230, Timestep 3878077,  Ep.Timesteps 1000, Score: 2022.81, Avg.Score: 945.42, Max.Score: 2197.62, Time: 38:38:31 \n",
      "Ep. 6240, Timestep 3888077,  Ep.Timesteps 1000, Score: 2249.44, Avg.Score: 1136.18, Max.Score: 2249.44, Time: 38:44:10 \n",
      "Ep. 6250, Timestep 3898077,  Ep.Timesteps 1000, Score: 2288.06, Avg.Score: 1347.49, Max.Score: 2313.20, Time: 38:49:50 \n",
      "Ep. 6260, Timestep 3908077,  Ep.Timesteps 1000, Score: 2076.71, Avg.Score: 1549.98, Max.Score: 2313.20, Time: 38:55:33 \n",
      "Ep. 6270, Timestep 3917430,  Ep.Timesteps 1000, Score: 2092.68, Avg.Score: 1735.55, Max.Score: 2313.20, Time: 39:00:51 \n",
      "Ep. 6280, Timestep 3926900,  Ep.Timesteps 1000, Score: 2008.92, Avg.Score: 1881.75, Max.Score: 2313.20, Time: 39:06:15 \n",
      "Ep. 6290, Timestep 3935922,  Ep.Timesteps 1000, Score: 2017.96, Avg.Score: 2022.98, Max.Score: 2313.20, Time: 39:11:21 \n",
      "Ep. 6300, Timestep 3945922,  Ep.Timesteps 1000, Score: 2168.67, Avg.Score: 2034.36, Max.Score: 2313.20, Time: 39:17:17 \n",
      "Ep. 6310, Timestep 3954933,  Ep.Timesteps 11, Score: 11.36, Avg.Score: 2038.76, Max.Score: 2313.20, Time: 39:23:31 \n",
      "Ep. 6320, Timestep 3963842,  Ep.Timesteps 1000, Score: 2118.21, Avg.Score: 2021.45, Max.Score: 2313.20, Time: 39:28:21 \n",
      "Ep. 6330, Timestep 3967080,  Ep.Timesteps 20, Score: 12.08, Avg.Score: 1877.19, Max.Score: 2313.20, Time: 39:30:51 \n",
      "Ep. 6340, Timestep 3969170,  Ep.Timesteps 14, Score: 7.83, Avg.Score: 1702.44, Max.Score: 2313.20, Time: 39:32:07 \n",
      "Ep. 6350, Timestep 3969321,  Ep.Timesteps 19, Score: 12.36, Avg.Score: 1483.85, Max.Score: 2267.96, Time: 39:32:13 \n",
      "Ep. 6360, Timestep 3969466,  Ep.Timesteps 12, Score: 8.20, Avg.Score: 1269.15, Max.Score: 2221.32, Time: 39:32:18 \n",
      "Ep. 6370, Timestep 3969602,  Ep.Timesteps 11, Score: 9.07, Avg.Score: 1076.37, Max.Score: 2221.32, Time: 39:32:24 \n",
      "Ep. 6380, Timestep 3969738,  Ep.Timesteps 14, Score: 9.99, Avg.Score: 884.31, Max.Score: 2221.32, Time: 39:32:29 \n",
      "Ep. 6390, Timestep 3969871,  Ep.Timesteps 12, Score: 6.86, Avg.Score: 702.04, Max.Score: 2221.32, Time: 39:32:33 \n",
      "Ep. 6400, Timestep 3969987,  Ep.Timesteps 13, Score: 9.32, Avg.Score: 496.46, Max.Score: 2221.32, Time: 39:32:38 \n",
      "Ep. 6410, Timestep 3970095,  Ep.Timesteps 12, Score: 8.95, Avg.Score: 302.78, Max.Score: 2197.25, Time: 39:32:42 \n",
      "Ep. 6420, Timestep 3970204,  Ep.Timesteps 11, Score: 7.88, Avg.Score: 113.73, Max.Score: 2138.05, Time: 39:32:46 \n",
      "Ep. 6430, Timestep 3970304,  Ep.Timesteps 10, Score: 8.22, Avg.Score: 48.76, Max.Score: 2082.70, Time: 39:32:50 \n",
      "Ep. 6440, Timestep 3970393,  Ep.Timesteps 8, Score: 6.87, Avg.Score: 8.18, Max.Score: 12.80, Time: 39:32:53 \n",
      "Ep. 6450, Timestep 3970482,  Ep.Timesteps 9, Score: 7.33, Avg.Score: 8.05, Max.Score: 12.80, Time: 39:32:57 \n",
      "Ep. 6460, Timestep 3970571,  Ep.Timesteps 9, Score: 8.14, Avg.Score: 7.97, Max.Score: 11.06, Time: 39:33:00 \n",
      "Ep. 6470, Timestep 3970655,  Ep.Timesteps 8, Score: 7.45, Avg.Score: 7.90, Max.Score: 11.06, Time: 39:33:03 \n",
      "Ep. 6480, Timestep 3970740,  Ep.Timesteps 9, Score: 7.19, Avg.Score: 7.81, Max.Score: 10.74, Time: 39:33:06 \n",
      "Ep. 6490, Timestep 3970825,  Ep.Timesteps 8, Score: 7.52, Avg.Score: 7.76, Max.Score: 9.54, Time: 39:33:10 \n",
      "Ep. 6500, Timestep 3970907,  Ep.Timesteps 9, Score: 8.99, Avg.Score: 7.70, Max.Score: 8.99, Time: 39:33:13 \n",
      "Ep. 6510, Timestep 3970991,  Ep.Timesteps 8, Score: 6.82, Avg.Score: 7.64, Max.Score: 8.99, Time: 39:33:16 \n",
      "Ep. 6520, Timestep 3971073,  Ep.Timesteps 8, Score: 6.59, Avg.Score: 7.56, Max.Score: 8.99, Time: 39:33:19 \n",
      "Ep. 6530, Timestep 3971154,  Ep.Timesteps 8, Score: 5.73, Avg.Score: 7.47, Max.Score: 8.99, Time: 39:33:22 \n",
      "Ep. 6540, Timestep 3971241,  Ep.Timesteps 9, Score: 6.52, Avg.Score: 7.39, Max.Score: 8.99, Time: 39:33:25 \n",
      "Ep. 6550, Timestep 3971344,  Ep.Timesteps 11, Score: 7.38, Avg.Score: 7.32, Max.Score: 8.99, Time: 39:33:29 \n",
      "Ep. 6560, Timestep 3971445,  Ep.Timesteps 11, Score: 4.63, Avg.Score: 7.18, Max.Score: 8.99, Time: 39:33:33 \n",
      "Ep. 6570, Timestep 3971544,  Ep.Timesteps 10, Score: 4.86, Avg.Score: 7.05, Max.Score: 8.99, Time: 39:33:36 \n",
      "Ep. 6580, Timestep 3971642,  Ep.Timesteps 11, Score: 5.57, Avg.Score: 6.89, Max.Score: 8.99, Time: 39:33:40 \n",
      "Ep. 6590, Timestep 3971746,  Ep.Timesteps 11, Score: 7.59, Avg.Score: 6.79, Max.Score: 8.99, Time: 39:33:44 \n",
      "Ep. 6600, Timestep 3971860,  Ep.Timesteps 11, Score: 3.41, Avg.Score: 6.56, Max.Score: 8.75, Time: 39:33:48 \n",
      "Ep. 6610, Timestep 3971984,  Ep.Timesteps 12, Score: 5.20, Avg.Score: 6.34, Max.Score: 8.75, Time: 39:33:53 \n",
      "Ep. 6620, Timestep 3972114,  Ep.Timesteps 13, Score: 6.27, Avg.Score: 6.21, Max.Score: 8.75, Time: 39:33:58 \n",
      "Ep. 6630, Timestep 3972245,  Ep.Timesteps 12, Score: 3.43, Avg.Score: 6.04, Max.Score: 8.75, Time: 39:34:02 \n",
      "Ep. 6640, Timestep 3972373,  Ep.Timesteps 12, Score: 5.15, Avg.Score: 5.92, Max.Score: 8.75, Time: 39:34:07 \n",
      "Ep. 6650, Timestep 3972496,  Ep.Timesteps 12, Score: 4.83, Avg.Score: 5.79, Max.Score: 8.75, Time: 39:34:12 \n",
      "Ep. 6660, Timestep 3972617,  Ep.Timesteps 12, Score: 4.45, Avg.Score: 5.66, Max.Score: 8.75, Time: 39:34:16 \n",
      "Ep. 6670, Timestep 3972743,  Ep.Timesteps 13, Score: 7.28, Avg.Score: 5.64, Max.Score: 8.75, Time: 39:34:21 \n",
      "Ep. 6680, Timestep 3972866,  Ep.Timesteps 12, Score: 3.73, Avg.Score: 5.56, Max.Score: 8.75, Time: 39:34:25 \n",
      "Ep. 6690, Timestep 3972992,  Ep.Timesteps 12, Score: 5.38, Avg.Score: 5.45, Max.Score: 8.36, Time: 39:34:30 \n",
      "Ep. 6700, Timestep 3973115,  Ep.Timesteps 12, Score: 5.33, Avg.Score: 5.48, Max.Score: 8.36, Time: 39:34:35 \n",
      "Ep. 6710, Timestep 3973234,  Ep.Timesteps 14, Score: 8.23, Avg.Score: 5.50, Max.Score: 8.36, Time: 39:34:39 \n",
      "Ep. 6720, Timestep 3973369,  Ep.Timesteps 12, Score: 6.76, Avg.Score: 5.60, Max.Score: 9.80, Time: 39:34:44 \n",
      "Ep. 6730, Timestep 3973488,  Ep.Timesteps 12, Score: 5.31, Avg.Score: 5.67, Max.Score: 9.80, Time: 39:34:49 \n",
      "Ep. 6740, Timestep 3973605,  Ep.Timesteps 12, Score: 5.69, Avg.Score: 5.68, Max.Score: 9.80, Time: 39:34:53 \n",
      "Ep. 6750, Timestep 3974721,  Ep.Timesteps 1000, Score: 2053.60, Avg.Score: 26.28, Max.Score: 2053.60, Time: 39:35:01 \n",
      "Ep. 6760, Timestep 3981470,  Ep.Timesteps 1000, Score: 2083.97, Avg.Score: 150.57, Max.Score: 2083.97, Time: 39:39:08 \n",
      "Ep. 6770, Timestep 3989245,  Ep.Timesteps 1000, Score: 2132.07, Avg.Score: 301.36, Max.Score: 2132.07, Time: 39:43:48 \n",
      "Ep. 6780, Timestep 3995675,  Ep.Timesteps 1000, Score: 2235.15, Avg.Score: 440.59, Max.Score: 2261.85, Time: 39:47:42 \n",
      "Ep. 6790, Timestep 4005675,  Ep.Timesteps 1000, Score: 2166.17, Avg.Score: 655.02, Max.Score: 2261.85, Time: 39:53:46 \n",
      "Ep. 6800, Timestep 4012169,  Ep.Timesteps 251, Score: 507.21, Avg.Score: 791.76, Max.Score: 2261.85, Time: 39:58:06 \n",
      "Ep. 6810, Timestep 4019558,  Ep.Timesteps 1000, Score: 1895.41, Avg.Score: 927.70, Max.Score: 2261.85, Time: 40:02:08 \n",
      "Ep. 6820, Timestep 4025823,  Ep.Timesteps 1000, Score: 1975.88, Avg.Score: 1054.65, Max.Score: 2261.85, Time: 40:05:54 \n",
      "Ep. 6830, Timestep 4035823,  Ep.Timesteps 1000, Score: 2133.99, Avg.Score: 1267.11, Max.Score: 2266.64, Time: 40:11:59 \n",
      "Ep. 6840, Timestep 4045823,  Ep.Timesteps 1000, Score: 2210.47, Avg.Score: 1487.21, Max.Score: 2266.64, Time: 40:17:57 \n",
      "Ep. 6850, Timestep 4055823,  Ep.Timesteps 1000, Score: 1955.77, Avg.Score: 1674.49, Max.Score: 2266.64, Time: 40:23:50 \n",
      "Ep. 6860, Timestep 4065823,  Ep.Timesteps 1000, Score: 2062.65, Avg.Score: 1763.39, Max.Score: 2266.64, Time: 40:29:39 \n",
      "Ep. 6870, Timestep 4075823,  Ep.Timesteps 1000, Score: 2039.59, Avg.Score: 1806.44, Max.Score: 2266.64, Time: 40:35:29 \n",
      "Ep. 6880, Timestep 4085119,  Ep.Timesteps 1000, Score: 2180.77, Avg.Score: 1865.96, Max.Score: 2266.64, Time: 40:40:56 \n",
      "Ep. 6890, Timestep 4095119,  Ep.Timesteps 1000, Score: 2171.32, Avg.Score: 1869.08, Max.Score: 2266.64, Time: 40:46:46 \n",
      "Ep. 6900, Timestep 4104856,  Ep.Timesteps 1000, Score: 2195.39, Avg.Score: 1944.03, Max.Score: 2266.64, Time: 40:52:29 \n",
      "Ep. 6910, Timestep 4114856,  Ep.Timesteps 1000, Score: 2289.86, Avg.Score: 2033.01, Max.Score: 2289.86, Time: 40:58:18 \n",
      "Ep. 6920, Timestep 4123983,  Ep.Timesteps 127, Score: 242.41, Avg.Score: 2111.96, Max.Score: 2326.61, Time: 41:04:06 \n",
      "Ep. 6930, Timestep 4133983,  Ep.Timesteps 1000, Score: 2242.70, Avg.Score: 2123.39, Max.Score: 2326.61, Time: 41:09:30 \n",
      "Ep. 6940, Timestep 4143983,  Ep.Timesteps 1000, Score: 2252.51, Avg.Score: 2127.76, Max.Score: 2326.61, Time: 41:15:19 \n",
      "Ep. 6950, Timestep 4153983,  Ep.Timesteps 1000, Score: 2285.08, Avg.Score: 2147.99, Max.Score: 2335.83, Time: 41:21:10 \n",
      "Ep. 6960, Timestep 4163983,  Ep.Timesteps 1000, Score: 2249.38, Avg.Score: 2161.67, Max.Score: 2335.83, Time: 41:27:05 \n",
      "Ep. 6970, Timestep 4173983,  Ep.Timesteps 1000, Score: 2207.48, Avg.Score: 2191.39, Max.Score: 2335.83, Time: 41:32:57 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 6980, Timestep 4182420,  Ep.Timesteps 1000, Score: 2302.95, Avg.Score: 2183.81, Max.Score: 2346.66, Time: 41:37:55 \n",
      "Ep. 6990, Timestep 4192420,  Ep.Timesteps 1000, Score: 2257.60, Avg.Score: 2192.24, Max.Score: 2346.66, Time: 41:43:45 \n",
      "Ep. 7000, Timestep 4201332,  Ep.Timesteps 1000, Score: 2263.31, Avg.Score: 2180.31, Max.Score: 2346.66, Time: 41:48:58 \n",
      "Ep. 7010, Timestep 4211332,  Ep.Timesteps 1000, Score: 2325.45, Avg.Score: 2182.71, Max.Score: 2346.66, Time: 41:54:56 \n",
      "Ep. 7020, Timestep 4220040,  Ep.Timesteps 1000, Score: 2324.21, Avg.Score: 2174.46, Max.Score: 2346.66, Time: 42:00:28 \n",
      "Ep. 7030, Timestep 4230040,  Ep.Timesteps 1000, Score: 2392.00, Avg.Score: 2179.65, Max.Score: 2392.00, Time: 42:06:52 \n",
      "Ep. 7040, Timestep 4239258,  Ep.Timesteps 1000, Score: 2100.26, Avg.Score: 2159.88, Max.Score: 2392.00, Time: 42:12:51 \n",
      "Ep. 7050, Timestep 4248977,  Ep.Timesteps 1000, Score: 2230.06, Avg.Score: 2145.60, Max.Score: 2392.00, Time: 42:19:07 \n",
      "Ep. 7060, Timestep 4253534,  Ep.Timesteps 108, Score: 179.05, Avg.Score: 2017.51, Max.Score: 2392.00, Time: 42:22:36 \n",
      "Ep. 7070, Timestep 4254158,  Ep.Timesteps 43, Score: 43.21, Avg.Score: 1801.45, Max.Score: 2392.00, Time: 42:23:02 \n",
      "Ep. 7080, Timestep 4255041,  Ep.Timesteps 112, Score: 183.31, Avg.Score: 1622.78, Max.Score: 2392.00, Time: 42:23:34 \n",
      "Ep. 7090, Timestep 4255651,  Ep.Timesteps 41, Score: 52.53, Avg.Score: 1403.16, Max.Score: 2392.00, Time: 42:24:04 \n",
      "Ep. 7100, Timestep 4256107,  Ep.Timesteps 33, Score: 40.56, Avg.Score: 1207.59, Max.Score: 2392.00, Time: 42:24:21 \n",
      "Ep. 7110, Timestep 4256580,  Ep.Timesteps 58, Score: 39.41, Avg.Score: 984.45, Max.Score: 2392.00, Time: 42:24:39 \n",
      "Ep. 7120, Timestep 4257249,  Ep.Timesteps 54, Score: 43.49, Avg.Score: 793.39, Max.Score: 2392.00, Time: 42:25:05 \n",
      "Ep. 7130, Timestep 4257841,  Ep.Timesteps 61, Score: 54.85, Avg.Score: 569.48, Max.Score: 2368.67, Time: 42:25:28 \n",
      "Ep. 7140, Timestep 4258374,  Ep.Timesteps 42, Score: 35.63, Avg.Score: 368.89, Max.Score: 2318.71, Time: 42:25:50 \n",
      "Ep. 7150, Timestep 4258994,  Ep.Timesteps 77, Score: 92.71, Avg.Score: 160.41, Max.Score: 2318.71, Time: 42:26:11 \n",
      "Ep. 7160, Timestep 4259760,  Ep.Timesteps 85, Score: 103.77, Avg.Score: 70.06, Max.Score: 196.35, Time: 42:26:39 \n",
      "Ep. 7170, Timestep 4260499,  Ep.Timesteps 59, Score: 58.16, Avg.Score: 69.89, Max.Score: 196.35, Time: 42:27:08 \n",
      "Ep. 7180, Timestep 4261296,  Ep.Timesteps 99, Score: 100.41, Avg.Score: 64.99, Max.Score: 196.35, Time: 42:27:37 \n",
      "Ep. 7190, Timestep 4262040,  Ep.Timesteps 64, Score: 63.37, Avg.Score: 64.94, Max.Score: 156.15, Time: 42:28:06 \n",
      "Ep. 7200, Timestep 4262818,  Ep.Timesteps 73, Score: 75.89, Avg.Score: 69.03, Max.Score: 156.15, Time: 42:28:38 \n",
      "Ep. 7210, Timestep 4263581,  Ep.Timesteps 77, Score: 91.48, Avg.Score: 72.97, Max.Score: 156.15, Time: 42:29:07 \n",
      "Ep. 7220, Timestep 4264464,  Ep.Timesteps 100, Score: 129.32, Avg.Score: 76.76, Max.Score: 129.32, Time: 42:29:42 \n",
      "Ep. 7230, Timestep 4265421,  Ep.Timesteps 95, Score: 119.00, Avg.Score: 83.38, Max.Score: 134.71, Time: 42:30:22 \n",
      "Ep. 7240, Timestep 4266436,  Ep.Timesteps 102, Score: 148.76, Avg.Score: 92.58, Max.Score: 160.71, Time: 42:31:00 \n",
      "Ep. 7250, Timestep 4267550,  Ep.Timesteps 134, Score: 205.92, Avg.Score: 103.30, Max.Score: 227.16, Time: 42:31:42 \n",
      "Ep. 7260, Timestep 4268682,  Ep.Timesteps 159, Score: 257.78, Avg.Score: 111.90, Max.Score: 257.78, Time: 42:32:25 \n",
      "Ep. 7270, Timestep 4270178,  Ep.Timesteps 83, Score: 120.07, Avg.Score: 127.95, Max.Score: 542.63, Time: 42:33:26 \n",
      "Ep. 7280, Timestep 4275375,  Ep.Timesteps 222, Score: 431.04, Avg.Score: 222.45, Max.Score: 2130.98, Time: 42:36:33 \n",
      "Ep. 7290, Timestep 4279021,  Ep.Timesteps 307, Score: 626.11, Avg.Score: 291.94, Max.Score: 2291.04, Time: 42:38:49 \n",
      "Ep. 7300, Timestep 4286866,  Ep.Timesteps 1000, Score: 2219.73, Avg.Score: 450.22, Max.Score: 2291.04, Time: 42:43:35 \n",
      "Ep. 7310, Timestep 4295649,  Ep.Timesteps 657, Score: 1388.45, Avg.Score: 628.66, Max.Score: 2291.04, Time: 42:49:20 \n",
      "Ep. 7320, Timestep 4303148,  Ep.Timesteps 374, Score: 800.41, Avg.Score: 777.56, Max.Score: 2291.04, Time: 42:54:42 \n",
      "Ep. 7330, Timestep 4309318,  Ep.Timesteps 1000, Score: 2176.39, Avg.Score: 898.58, Max.Score: 2291.04, Time: 42:58:22 \n",
      "Ep. 7340, Timestep 4315479,  Ep.Timesteps 1000, Score: 2275.94, Avg.Score: 1019.67, Max.Score: 2291.04, Time: 43:02:32 \n",
      "Ep. 7350, Timestep 4325325,  Ep.Timesteps 1000, Score: 2127.07, Avg.Score: 1215.67, Max.Score: 2307.40, Time: 43:09:09 \n",
      "Ep. 7360, Timestep 4334449,  Ep.Timesteps 1000, Score: 2172.43, Avg.Score: 1391.34, Max.Score: 2307.40, Time: 43:15:22 \n",
      "Ep. 7370, Timestep 4344449,  Ep.Timesteps 1000, Score: 2152.35, Avg.Score: 1577.70, Max.Score: 2307.40, Time: 43:21:52 \n",
      "Ep. 7380, Timestep 4353505,  Ep.Timesteps 1000, Score: 2108.89, Avg.Score: 1665.18, Max.Score: 2307.40, Time: 43:27:53 \n",
      "Ep. 7390, Timestep 4363505,  Ep.Timesteps 1000, Score: 2100.70, Avg.Score: 1799.21, Max.Score: 2307.40, Time: 43:34:27 \n",
      "Ep. 7400, Timestep 4373505,  Ep.Timesteps 1000, Score: 2011.91, Avg.Score: 1841.97, Max.Score: 2307.40, Time: 43:40:57 \n",
      "Ep. 7410, Timestep 4381657,  Ep.Timesteps 79, Score: 81.43, Avg.Score: 1818.75, Max.Score: 2307.40, Time: 43:46:24 \n",
      "Ep. 7420, Timestep 4390751,  Ep.Timesteps 1000, Score: 2165.24, Avg.Score: 1850.59, Max.Score: 2307.40, Time: 43:51:16 \n",
      "Ep. 7430, Timestep 4400751,  Ep.Timesteps 1000, Score: 2013.74, Avg.Score: 1922.84, Max.Score: 2307.40, Time: 43:57:12 \n",
      "Ep. 7440, Timestep 4410120,  Ep.Timesteps 1000, Score: 2050.31, Avg.Score: 1982.02, Max.Score: 2307.40, Time: 44:02:42 \n",
      "Ep. 7450, Timestep 4420120,  Ep.Timesteps 1000, Score: 2077.15, Avg.Score: 1978.88, Max.Score: 2239.57, Time: 44:08:48 \n",
      "Ep. 7460, Timestep 4430120,  Ep.Timesteps 1000, Score: 2241.26, Avg.Score: 2005.13, Max.Score: 2294.69, Time: 44:14:49 \n",
      "Ep. 7470, Timestep 4439597,  Ep.Timesteps 1000, Score: 2157.56, Avg.Score: 2001.98, Max.Score: 2302.38, Time: 44:20:40 \n",
      "Ep. 7480, Timestep 4449597,  Ep.Timesteps 1000, Score: 2174.94, Avg.Score: 2036.04, Max.Score: 2302.38, Time: 44:27:30 \n",
      "Ep. 7490, Timestep 4456679,  Ep.Timesteps 1000, Score: 2113.60, Avg.Score: 1979.10, Max.Score: 2306.28, Time: 44:32:32 \n",
      "Ep. 7500, Timestep 4465925,  Ep.Timesteps 246, Score: 511.49, Avg.Score: 1972.58, Max.Score: 2306.28, Time: 44:38:42 \n",
      "Ep. 7510, Timestep 4474160,  Ep.Timesteps 1000, Score: 2303.97, Avg.Score: 1996.78, Max.Score: 2373.31, Time: 44:43:16 \n",
      "Ep. 7520, Timestep 4482549,  Ep.Timesteps 1000, Score: 2215.27, Avg.Score: 1994.58, Max.Score: 2373.31, Time: 44:48:19 \n",
      "Ep. 7530, Timestep 4491877,  Ep.Timesteps 1000, Score: 2330.36, Avg.Score: 2005.79, Max.Score: 2432.52, Time: 44:54:04 \n",
      "Ep. 7540, Timestep 4501199,  Ep.Timesteps 1000, Score: 2320.11, Avg.Score: 2025.50, Max.Score: 2432.52, Time: 44:59:42 \n",
      "Ep. 7550, Timestep 4510371,  Ep.Timesteps 1000, Score: 2362.45, Avg.Score: 2029.77, Max.Score: 2432.52, Time: 45:05:08 \n",
      "Ep. 7560, Timestep 4519553,  Ep.Timesteps 1000, Score: 2288.01, Avg.Score: 2021.10, Max.Score: 2432.52, Time: 45:10:35 \n",
      "Ep. 7570, Timestep 4528912,  Ep.Timesteps 1000, Score: 2290.81, Avg.Score: 2027.87, Max.Score: 2432.52, Time: 45:16:05 \n",
      "Ep. 7580, Timestep 4538912,  Ep.Timesteps 1000, Score: 2276.00, Avg.Score: 2033.46, Max.Score: 2432.52, Time: 45:22:15 \n",
      "Ep. 7590, Timestep 4548716,  Ep.Timesteps 1000, Score: 2150.20, Avg.Score: 2097.82, Max.Score: 2432.52, Time: 45:28:12 \n",
      "Ep. 7600, Timestep 4556975,  Ep.Timesteps 341, Score: 636.43, Avg.Score: 2065.66, Max.Score: 2432.52, Time: 45:33:31 \n",
      "Ep. 7610, Timestep 4557965,  Ep.Timesteps 63, Score: 86.73, Avg.Score: 1891.46, Max.Score: 2432.52, Time: 45:34:16 \n",
      "Ep. 7620, Timestep 4559000,  Ep.Timesteps 40, Score: 57.34, Avg.Score: 1716.55, Max.Score: 2432.52, Time: 45:34:53 \n",
      "Ep. 7630, Timestep 4568218,  Ep.Timesteps 1000, Score: 2162.15, Avg.Score: 1680.62, Max.Score: 2398.30, Time: 45:40:00 \n",
      "Ep. 7640, Timestep 4578218,  Ep.Timesteps 1000, Score: 2255.63, Avg.Score: 1687.97, Max.Score: 2398.30, Time: 45:46:03 \n",
      "Ep. 7650, Timestep 4588218,  Ep.Timesteps 1000, Score: 2283.77, Avg.Score: 1698.88, Max.Score: 2394.52, Time: 45:52:13 \n",
      "Ep. 7660, Timestep 4597393,  Ep.Timesteps 1000, Score: 2313.86, Avg.Score: 1694.38, Max.Score: 2394.52, Time: 45:57:28 \n",
      "Ep. 7670, Timestep 4606757,  Ep.Timesteps 1000, Score: 2269.65, Avg.Score: 1697.39, Max.Score: 2397.64, Time: 46:02:49 \n",
      "Ep. 7680, Timestep 4614365,  Ep.Timesteps 1000, Score: 2272.32, Avg.Score: 1638.83, Max.Score: 2397.64, Time: 46:07:22 \n",
      "Ep. 7690, Timestep 4622567,  Ep.Timesteps 39, Score: 53.23, Avg.Score: 1606.82, Max.Score: 2397.64, Time: 46:12:33 \n",
      "Ep. 7700, Timestep 4631606,  Ep.Timesteps 1000, Score: 2327.20, Avg.Score: 1645.09, Max.Score: 2397.64, Time: 46:17:12 \n",
      "Ep. 7710, Timestep 4640498,  Ep.Timesteps 1000, Score: 2335.17, Avg.Score: 1836.55, Max.Score: 2397.64, Time: 46:22:18 \n",
      "Ep. 7720, Timestep 4650498,  Ep.Timesteps 1000, Score: 2291.42, Avg.Score: 2054.50, Max.Score: 2397.64, Time: 46:27:59 \n",
      "Ep. 7730, Timestep 4660498,  Ep.Timesteps 1000, Score: 2232.34, Avg.Score: 2104.42, Max.Score: 2397.64, Time: 46:33:40 \n",
      "Ep. 7740, Timestep 4669613,  Ep.Timesteps 115, Score: 180.93, Avg.Score: 2094.07, Max.Score: 2397.64, Time: 46:39:21 \n",
      "Ep. 7750, Timestep 4678947,  Ep.Timesteps 1000, Score: 2384.20, Avg.Score: 2086.67, Max.Score: 2397.92, Time: 46:44:12 \n",
      "Ep. 7760, Timestep 4688058,  Ep.Timesteps 1000, Score: 2379.86, Avg.Score: 2094.21, Max.Score: 2397.92, Time: 46:49:23 \n",
      "Ep. 7770, Timestep 4696781,  Ep.Timesteps 1000, Score: 2374.51, Avg.Score: 2081.37, Max.Score: 2401.44, Time: 46:54:23 \n",
      "Ep. 7780, Timestep 4706781,  Ep.Timesteps 1000, Score: 2327.56, Avg.Score: 2143.55, Max.Score: 2401.44, Time: 47:00:04 \n",
      "Ep. 7790, Timestep 4715323,  Ep.Timesteps 69, Score: 104.85, Avg.Score: 2157.76, Max.Score: 2401.44, Time: 47:05:25 \n",
      "Ep. 7800, Timestep 4724663,  Ep.Timesteps 1000, Score: 2424.13, Avg.Score: 2166.43, Max.Score: 2424.13, Time: 47:10:17 \n",
      "Ep. 7810, Timestep 4734663,  Ep.Timesteps 1000, Score: 2376.27, Avg.Score: 2198.88, Max.Score: 2433.58, Time: 47:15:58 \n",
      "Ep. 7820, Timestep 4744663,  Ep.Timesteps 1000, Score: 2391.22, Avg.Score: 2204.80, Max.Score: 2433.58, Time: 47:21:41 \n",
      "Ep. 7830, Timestep 4754663,  Ep.Timesteps 1000, Score: 2304.42, Avg.Score: 2212.88, Max.Score: 2433.58, Time: 47:27:23 \n",
      "Ep. 7840, Timestep 4762967,  Ep.Timesteps 127, Score: 241.64, Avg.Score: 2199.31, Max.Score: 2452.30, Time: 47:32:34 \n",
      "Ep. 7850, Timestep 4772584,  Ep.Timesteps 617, Score: 1430.93, Avg.Score: 2209.22, Max.Score: 2452.30, Time: 47:38:18 \n",
      "Ep. 7860, Timestep 4782584,  Ep.Timesteps 1000, Score: 2375.33, Avg.Score: 2234.20, Max.Score: 2452.30, Time: 47:43:57 \n",
      "Ep. 7870, Timestep 4792329,  Ep.Timesteps 1000, Score: 2411.95, Avg.Score: 2259.48, Max.Score: 2452.30, Time: 47:49:42 \n",
      "Ep. 7880, Timestep 4801819,  Ep.Timesteps 1000, Score: 2347.34, Avg.Score: 2246.34, Max.Score: 2452.30, Time: 47:53:44 \n",
      "Ep. 7890, Timestep 4811819,  Ep.Timesteps 1000, Score: 2329.86, Avg.Score: 2282.44, Max.Score: 2452.30, Time: 47:57:42 \n",
      "Ep. 7900, Timestep 4821430,  Ep.Timesteps 1000, Score: 2247.99, Avg.Score: 2284.69, Max.Score: 2452.30, Time: 48:01:30 \n",
      "Ep. 7910, Timestep 4823512,  Ep.Timesteps 10, Score: 2.71, Avg.Score: 2092.01, Max.Score: 2452.30, Time: 48:02:41 \n",
      "Ep. 7920, Timestep 4823618,  Ep.Timesteps 11, Score: 2.19, Avg.Score: 1853.79, Max.Score: 2452.30, Time: 48:02:44 \n",
      "Ep. 7930, Timestep 4823723,  Ep.Timesteps 10, Score: 3.70, Avg.Score: 1615.22, Max.Score: 2452.30, Time: 48:02:47 \n",
      "Ep. 7940, Timestep 4823832,  Ep.Timesteps 10, Score: 5.05, Avg.Score: 1418.34, Max.Score: 2439.42, Time: 48:02:49 \n",
      "Ep. 7950, Timestep 4823938,  Ep.Timesteps 11, Score: 3.21, Avg.Score: 1191.49, Max.Score: 2432.27, Time: 48:02:52 \n",
      "Ep. 7960, Timestep 4824047,  Ep.Timesteps 12, Score: 3.27, Avg.Score: 953.03, Max.Score: 2425.93, Time: 48:02:55 \n",
      "Ep. 7970, Timestep 4825158,  Ep.Timesteps 13, Score: 3.87, Avg.Score: 744.84, Max.Score: 2425.93, Time: 48:03:22 \n",
      "Ep. 7980, Timestep 4828395,  Ep.Timesteps 164, Score: 327.28, Avg.Score: 595.14, Max.Score: 2425.93, Time: 48:04:35 \n",
      "Ep. 7990, Timestep 4837366,  Ep.Timesteps 1000, Score: 2419.09, Avg.Score: 568.26, Max.Score: 2425.93, Time: 48:07:52 \n",
      "Ep. 8000, Timestep 4846079,  Ep.Timesteps 1000, Score: 2404.17, Avg.Score: 554.40, Max.Score: 2432.31, Time: 48:11:15 \n",
      "Ep. 8010, Timestep 4855298,  Ep.Timesteps 219, Score: 455.48, Avg.Score: 729.42, Max.Score: 2485.28, Time: 48:15:09 \n",
      "Ep. 8020, Timestep 4864834,  Ep.Timesteps 536, Score: 1264.96, Avg.Score: 957.84, Max.Score: 2485.28, Time: 48:18:46 \n",
      "Ep. 8030, Timestep 4874615,  Ep.Timesteps 1000, Score: 2366.44, Avg.Score: 1190.37, Max.Score: 2485.28, Time: 48:22:27 \n",
      "Ep. 8040, Timestep 4884226,  Ep.Timesteps 1000, Score: 2314.03, Avg.Score: 1416.59, Max.Score: 2485.28, Time: 48:26:10 \n",
      "Ep. 8050, Timestep 4893585,  Ep.Timesteps 1000, Score: 2428.14, Avg.Score: 1641.81, Max.Score: 2485.28, Time: 48:29:48 \n",
      "Ep. 8060, Timestep 4903585,  Ep.Timesteps 1000, Score: 2452.74, Avg.Score: 1881.68, Max.Score: 2485.28, Time: 48:33:41 \n",
      "Ep. 8070, Timestep 4913585,  Ep.Timesteps 1000, Score: 2436.77, Avg.Score: 2100.62, Max.Score: 2485.28, Time: 48:37:36 \n",
      "Ep. 8080, Timestep 4923585,  Ep.Timesteps 1000, Score: 2306.81, Avg.Score: 2268.75, Max.Score: 2485.28, Time: 48:41:29 \n",
      "Ep. 8090, Timestep 4933585,  Ep.Timesteps 1000, Score: 2393.11, Avg.Score: 2295.83, Max.Score: 2485.28, Time: 48:45:23 \n",
      "Ep. 8100, Timestep 4942663,  Ep.Timesteps 1000, Score: 2358.01, Avg.Score: 2296.75, Max.Score: 2485.28, Time: 48:48:55 \n",
      "Ep. 8110, Timestep 4952663,  Ep.Timesteps 1000, Score: 2474.75, Avg.Score: 2317.18, Max.Score: 2474.75, Time: 48:52:51 \n",
      "Ep. 8120, Timestep 4962663,  Ep.Timesteps 1000, Score: 2362.13, Avg.Score: 2330.73, Max.Score: 2480.17, Time: 48:56:45 \n",
      "Ep. 8130, Timestep 4971771,  Ep.Timesteps 1000, Score: 2467.04, Avg.Score: 2321.01, Max.Score: 2536.06, Time: 49:00:22 \n",
      "Ep. 8140, Timestep 4981771,  Ep.Timesteps 1000, Score: 2451.27, Avg.Score: 2339.10, Max.Score: 2536.06, Time: 49:04:21 \n",
      "Ep. 8150, Timestep 4991771,  Ep.Timesteps 1000, Score: 2406.90, Avg.Score: 2355.07, Max.Score: 2536.06, Time: 49:08:14 \n",
      "Ep. 8160, Timestep 5001771,  Ep.Timesteps 1000, Score: 2426.56, Avg.Score: 2357.36, Max.Score: 2536.06, Time: 49:12:06 \n",
      "Ep. 8170, Timestep 5011771,  Ep.Timesteps 1000, Score: 2457.95, Avg.Score: 2361.11, Max.Score: 2536.06, Time: 49:16:12 \n",
      "Ep. 8180, Timestep 5021771,  Ep.Timesteps 1000, Score: 2471.86, Avg.Score: 2365.16, Max.Score: 2536.06, Time: 49:20:06 \n",
      "Ep. 8190, Timestep 5030939,  Ep.Timesteps 406, Score: 951.22, Avg.Score: 2351.37, Max.Score: 2536.06, Time: 49:23:52 \n",
      "Ep. 8200, Timestep 5040610,  Ep.Timesteps 1000, Score: 2349.50, Avg.Score: 2375.97, Max.Score: 2536.06, Time: 49:27:24 \n",
      "Ep. 8210, Timestep 5050610,  Ep.Timesteps 1000, Score: 2368.01, Avg.Score: 2378.58, Max.Score: 2536.06, Time: 49:31:16 \n",
      "Ep. 8220, Timestep 5060610,  Ep.Timesteps 1000, Score: 2438.76, Avg.Score: 2381.63, Max.Score: 2536.06, Time: 49:35:06 \n",
      "Ep. 8230, Timestep 5069657,  Ep.Timesteps 47, Score: 73.55, Avg.Score: 2380.45, Max.Score: 2507.37, Time: 49:38:57 \n",
      "Ep. 8240, Timestep 5079657,  Ep.Timesteps 1000, Score: 2465.01, Avg.Score: 2379.61, Max.Score: 2507.37, Time: 49:42:26 \n",
      "Ep. 8250, Timestep 5087526,  Ep.Timesteps 1000, Score: 2401.89, Avg.Score: 2329.71, Max.Score: 2541.74, Time: 49:45:27 \n",
      "Ep. 8260, Timestep 5096874,  Ep.Timesteps 1000, Score: 2260.15, Avg.Score: 2300.84, Max.Score: 2541.74, Time: 49:49:02 \n",
      "Ep. 8270, Timestep 5106874,  Ep.Timesteps 1000, Score: 2295.99, Avg.Score: 2284.24, Max.Score: 2541.74, Time: 49:52:54 \n",
      "Ep. 8280, Timestep 5116874,  Ep.Timesteps 1000, Score: 2478.91, Avg.Score: 2285.33, Max.Score: 2541.74, Time: 49:56:44 \n",
      "Ep. 8290, Timestep 5126874,  Ep.Timesteps 1000, Score: 2434.49, Avg.Score: 2311.72, Max.Score: 2543.94, Time: 50:00:43 \n",
      "Ep. 8300, Timestep 5135982,  Ep.Timesteps 1000, Score: 2428.11, Avg.Score: 2298.74, Max.Score: 2543.94, Time: 50:04:18 \n",
      "Ep. 8310, Timestep 5145982,  Ep.Timesteps 1000, Score: 2430.45, Avg.Score: 2296.69, Max.Score: 2543.94, Time: 50:08:10 \n",
      "Ep. 8320, Timestep 5155982,  Ep.Timesteps 1000, Score: 2446.15, Avg.Score: 2294.90, Max.Score: 2543.94, Time: 50:12:00 \n",
      "Ep. 8330, Timestep 5165982,  Ep.Timesteps 1000, Score: 2363.63, Avg.Score: 2316.30, Max.Score: 2543.94, Time: 50:16:17 \n",
      "Ep. 8340, Timestep 5175982,  Ep.Timesteps 1000, Score: 2486.26, Avg.Score: 2314.98, Max.Score: 2543.94, Time: 50:20:19 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 8350, Timestep 5182517,  Ep.Timesteps 742, Score: 1765.31, Avg.Score: 2280.53, Max.Score: 2543.94, Time: 50:23:04 \n",
      "Ep. 8360, Timestep 5191043,  Ep.Timesteps 1000, Score: 2468.54, Avg.Score: 2273.74, Max.Score: 2543.94, Time: 50:26:17 \n",
      "Ep. 8370, Timestep 5201043,  Ep.Timesteps 1000, Score: 2435.98, Avg.Score: 2288.12, Max.Score: 2543.94, Time: 50:30:09 \n",
      "Ep. 8380, Timestep 5211043,  Ep.Timesteps 1000, Score: 2171.20, Avg.Score: 2276.12, Max.Score: 2543.94, Time: 50:34:01 \n",
      "Ep. 8390, Timestep 5221043,  Ep.Timesteps 1000, Score: 2247.60, Avg.Score: 2260.01, Max.Score: 2510.92, Time: 50:37:55 \n",
      "Ep. 8400, Timestep 5225764,  Ep.Timesteps 1000, Score: 1982.53, Avg.Score: 2122.80, Max.Score: 2510.92, Time: 50:39:44 \n",
      "Ep. 8410, Timestep 5227427,  Ep.Timesteps 157, Score: 227.38, Avg.Score: 1907.44, Max.Score: 2510.92, Time: 50:40:41 \n",
      "Ep. 8420, Timestep 5234772,  Ep.Timesteps 1000, Score: 1728.00, Avg.Score: 1800.77, Max.Score: 2510.92, Time: 50:43:13 \n",
      "Ep. 8430, Timestep 5243924,  Ep.Timesteps 1000, Score: 2170.38, Avg.Score: 1738.55, Max.Score: 2510.92, Time: 50:46:44 \n",
      "Ep. 8440, Timestep 5253924,  Ep.Timesteps 1000, Score: 2541.06, Avg.Score: 1732.70, Max.Score: 2541.06, Time: 50:50:36 \n",
      "Ep. 8450, Timestep 5263924,  Ep.Timesteps 1000, Score: 1875.53, Avg.Score: 1785.97, Max.Score: 2541.06, Time: 50:54:29 \n",
      "Ep. 8460, Timestep 5269237,  Ep.Timesteps 1000, Score: 2186.03, Avg.Score: 1686.71, Max.Score: 2541.06, Time: 50:56:32 \n",
      "Ep. 8470, Timestep 5279237,  Ep.Timesteps 1000, Score: 2429.94, Avg.Score: 1668.09, Max.Score: 2541.06, Time: 51:00:23 \n",
      "Ep. 8480, Timestep 5288357,  Ep.Timesteps 1000, Score: 2273.24, Avg.Score: 1648.48, Max.Score: 2541.06, Time: 51:03:54 \n",
      "Ep. 8490, Timestep 5298298,  Ep.Timesteps 1000, Score: 2429.34, Avg.Score: 1655.23, Max.Score: 2541.06, Time: 51:07:53 \n",
      "Ep. 8500, Timestep 5307957,  Ep.Timesteps 1000, Score: 2429.89, Avg.Score: 1803.31, Max.Score: 2541.06, Time: 51:11:36 \n",
      "Ep. 8510, Timestep 5317957,  Ep.Timesteps 1000, Score: 2420.95, Avg.Score: 2016.89, Max.Score: 2541.06, Time: 51:15:34 \n",
      "Ep. 8520, Timestep 5326987,  Ep.Timesteps 1000, Score: 2471.73, Avg.Score: 2099.61, Max.Score: 2541.06, Time: 51:19:17 \n",
      "Ep. 8530, Timestep 5336987,  Ep.Timesteps 1000, Score: 2401.94, Avg.Score: 2164.17, Max.Score: 2541.06, Time: 51:23:28 \n",
      "Ep. 8540, Timestep 5346987,  Ep.Timesteps 1000, Score: 2355.49, Avg.Score: 2172.10, Max.Score: 2569.33, Time: 51:27:27 \n",
      "Ep. 8550, Timestep 5356987,  Ep.Timesteps 1000, Score: 2400.63, Avg.Score: 2201.52, Max.Score: 2569.33, Time: 51:31:19 \n",
      "Ep. 8560, Timestep 5366987,  Ep.Timesteps 1000, Score: 2503.28, Avg.Score: 2340.40, Max.Score: 2569.33, Time: 51:35:12 \n",
      "Ep. 8570, Timestep 5376322,  Ep.Timesteps 1000, Score: 2497.26, Avg.Score: 2347.21, Max.Score: 2569.33, Time: 51:38:51 \n",
      "Ep. 8580, Timestep 5386322,  Ep.Timesteps 1000, Score: 2460.36, Avg.Score: 2380.53, Max.Score: 2569.33, Time: 51:42:42 \n",
      "Ep. 8590, Timestep 5395152,  Ep.Timesteps 1000, Score: 2519.16, Avg.Score: 2359.34, Max.Score: 2569.33, Time: 51:46:07 \n",
      "Ep. 8600, Timestep 5405152,  Ep.Timesteps 1000, Score: 2542.48, Avg.Score: 2375.33, Max.Score: 2569.33, Time: 51:49:59 \n",
      "Ep. 8610, Timestep 5415152,  Ep.Timesteps 1000, Score: 2568.94, Avg.Score: 2383.84, Max.Score: 2569.33, Time: 51:53:53 \n",
      "Ep. 8620, Timestep 5425152,  Ep.Timesteps 1000, Score: 2358.24, Avg.Score: 2407.30, Max.Score: 2569.33, Time: 51:57:45 \n",
      "Ep. 8630, Timestep 5433602,  Ep.Timesteps 926, Score: 2272.84, Avg.Score: 2366.45, Max.Score: 2569.33, Time: 52:01:03 \n",
      "Ep. 8640, Timestep 5442443,  Ep.Timesteps 1000, Score: 2513.16, Avg.Score: 2332.51, Max.Score: 2568.94, Time: 52:04:26 \n",
      "Ep. 8650, Timestep 5452443,  Ep.Timesteps 1000, Score: 2325.37, Avg.Score: 2337.00, Max.Score: 2568.94, Time: 52:08:22 \n",
      "Ep. 8660, Timestep 5462443,  Ep.Timesteps 1000, Score: 2494.27, Avg.Score: 2335.40, Max.Score: 2568.94, Time: 52:12:23 \n",
      "Ep. 8670, Timestep 5471273,  Ep.Timesteps 1000, Score: 2495.10, Avg.Score: 2324.91, Max.Score: 2568.94, Time: 52:15:54 \n",
      "Ep. 8680, Timestep 5480442,  Ep.Timesteps 1000, Score: 2421.48, Avg.Score: 2302.72, Max.Score: 2568.94, Time: 52:19:27 \n",
      "Ep. 8690, Timestep 5489708,  Ep.Timesteps 1000, Score: 2532.44, Avg.Score: 2313.53, Max.Score: 2568.94, Time: 52:23:09 \n",
      "Ep. 8700, Timestep 5498702,  Ep.Timesteps 393, Score: 867.66, Avg.Score: 2287.14, Max.Score: 2568.94, Time: 52:26:56 \n",
      "Ep. 8710, Timestep 5507817,  Ep.Timesteps 1000, Score: 2523.87, Avg.Score: 2268.31, Max.Score: 2563.07, Time: 52:30:21 \n",
      "Ep. 8720, Timestep 5516155,  Ep.Timesteps 1000, Score: 2559.79, Avg.Score: 2234.53, Max.Score: 2603.92, Time: 52:33:36 \n",
      "Ep. 8730, Timestep 5526155,  Ep.Timesteps 1000, Score: 2432.12, Avg.Score: 2279.25, Max.Score: 2603.92, Time: 52:37:37 \n",
      "Ep. 8740, Timestep 5536155,  Ep.Timesteps 1000, Score: 2552.21, Avg.Score: 2318.98, Max.Score: 2612.75, Time: 52:41:39 \n",
      "Ep. 8750, Timestep 5546155,  Ep.Timesteps 1000, Score: 2514.40, Avg.Score: 2331.46, Max.Score: 2636.63, Time: 52:45:35 \n",
      "Ep. 8760, Timestep 5555238,  Ep.Timesteps 1000, Score: 2607.45, Avg.Score: 2313.25, Max.Score: 2636.63, Time: 52:49:09 \n",
      "Ep. 8770, Timestep 5565226,  Ep.Timesteps 1000, Score: 2589.91, Avg.Score: 2347.32, Max.Score: 2636.63, Time: 52:53:07 \n",
      "Ep. 8780, Timestep 5575226,  Ep.Timesteps 1000, Score: 2349.02, Avg.Score: 2372.67, Max.Score: 2636.63, Time: 52:57:02 \n",
      "Ep. 8790, Timestep 5584241,  Ep.Timesteps 1000, Score: 2417.56, Avg.Score: 2364.81, Max.Score: 2636.63, Time: 53:00:34 \n",
      "Ep. 8800, Timestep 5594241,  Ep.Timesteps 1000, Score: 2479.67, Avg.Score: 2397.11, Max.Score: 2636.63, Time: 53:04:36 \n",
      "Ep. 8810, Timestep 5604241,  Ep.Timesteps 1000, Score: 2614.80, Avg.Score: 2415.24, Max.Score: 2636.63, Time: 53:08:37 \n",
      "Ep. 8820, Timestep 5611956,  Ep.Timesteps 1000, Score: 2541.23, Avg.Score: 2394.89, Max.Score: 2636.63, Time: 53:11:42 \n",
      "Ep. 8830, Timestep 5621449,  Ep.Timesteps 1000, Score: 2539.20, Avg.Score: 2384.83, Max.Score: 2636.63, Time: 53:15:32 \n",
      "Ep. 8840, Timestep 5631449,  Ep.Timesteps 1000, Score: 2370.44, Avg.Score: 2385.89, Max.Score: 2647.13, Time: 53:19:35 \n",
      "Ep. 8850, Timestep 5641449,  Ep.Timesteps 1000, Score: 2692.75, Avg.Score: 2380.82, Max.Score: 2692.75, Time: 53:23:43 \n",
      "Ep. 8860, Timestep 5650471,  Ep.Timesteps 1000, Score: 2414.00, Avg.Score: 2363.99, Max.Score: 2692.75, Time: 53:27:16 \n",
      "Ep. 8870, Timestep 5658860,  Ep.Timesteps 320, Score: 746.35, Avg.Score: 2317.87, Max.Score: 2692.75, Time: 53:30:48 \n",
      "Ep. 8880, Timestep 5668535,  Ep.Timesteps 1000, Score: 2575.79, Avg.Score: 2314.26, Max.Score: 2692.75, Time: 53:34:22 \n",
      "Ep. 8890, Timestep 5677652,  Ep.Timesteps 1000, Score: 2498.34, Avg.Score: 2324.46, Max.Score: 2692.75, Time: 53:38:16 \n",
      "Ep. 8900, Timestep 5686808,  Ep.Timesteps 1000, Score: 2436.09, Avg.Score: 2296.09, Max.Score: 2692.75, Time: 53:42:03 \n",
      "Ep. 8910, Timestep 5693894,  Ep.Timesteps 1000, Score: 2467.42, Avg.Score: 2220.24, Max.Score: 2692.75, Time: 53:45:06 \n",
      "Ep. 8920, Timestep 5698416,  Ep.Timesteps 316, Score: 739.86, Avg.Score: 2136.46, Max.Score: 2692.75, Time: 53:47:12 \n",
      "Ep. 8930, Timestep 5702939,  Ep.Timesteps 1000, Score: 2325.02, Avg.Score: 2003.13, Max.Score: 2692.75, Time: 53:49:50 \n",
      "Ep. 8940, Timestep 5712837,  Ep.Timesteps 1000, Score: 2530.51, Avg.Score: 1998.61, Max.Score: 2692.75, Time: 53:54:10 \n",
      "Ep. 8950, Timestep 5722553,  Ep.Timesteps 1000, Score: 2499.89, Avg.Score: 1990.76, Max.Score: 2637.76, Time: 53:58:10 \n",
      "Ep. 8960, Timestep 5728825,  Ep.Timesteps 1000, Score: 2413.82, Avg.Score: 1931.65, Max.Score: 2621.39, Time: 54:00:44 \n",
      "Ep. 8970, Timestep 5738825,  Ep.Timesteps 1000, Score: 2606.63, Avg.Score: 1980.32, Max.Score: 2659.51, Time: 54:04:51 \n",
      "Ep. 8980, Timestep 5747873,  Ep.Timesteps 1000, Score: 2371.05, Avg.Score: 1961.69, Max.Score: 2659.51, Time: 54:08:46 \n",
      "Ep. 8990, Timestep 5757873,  Ep.Timesteps 1000, Score: 2557.86, Avg.Score: 1987.71, Max.Score: 2659.51, Time: 54:12:42 \n",
      "Ep. 9000, Timestep 5767873,  Ep.Timesteps 1000, Score: 2555.76, Avg.Score: 2019.81, Max.Score: 2659.51, Time: 54:16:40 \n",
      "Ep. 9010, Timestep 5777873,  Ep.Timesteps 1000, Score: 2473.99, Avg.Score: 2104.75, Max.Score: 2659.51, Time: 54:20:37 \n",
      "Ep. 9020, Timestep 5787873,  Ep.Timesteps 1000, Score: 2590.67, Avg.Score: 2257.63, Max.Score: 2659.51, Time: 54:24:37 \n",
      "Ep. 9030, Timestep 5794973,  Ep.Timesteps 1000, Score: 2596.93, Avg.Score: 2333.13, Max.Score: 2661.23, Time: 54:27:35 \n",
      "Ep. 9040, Timestep 5804059,  Ep.Timesteps 1000, Score: 2553.98, Avg.Score: 2317.63, Max.Score: 2661.23, Time: 54:31:16 \n",
      "Ep. 9050, Timestep 5814059,  Ep.Timesteps 1000, Score: 2405.16, Avg.Score: 2325.82, Max.Score: 2661.23, Time: 54:35:21 \n",
      "Ep. 9060, Timestep 5822380,  Ep.Timesteps 1000, Score: 2493.95, Avg.Score: 2380.45, Max.Score: 2661.23, Time: 54:38:38 \n",
      "Ep. 9070, Timestep 5832380,  Ep.Timesteps 1000, Score: 2506.82, Avg.Score: 2377.74, Max.Score: 2661.23, Time: 54:42:34 \n",
      "Ep. 9080, Timestep 5841078,  Ep.Timesteps 1000, Score: 2508.92, Avg.Score: 2371.25, Max.Score: 2661.23, Time: 54:45:59 \n",
      "Ep. 9090, Timestep 5850213,  Ep.Timesteps 1000, Score: 2618.96, Avg.Score: 2350.36, Max.Score: 2661.23, Time: 54:49:33 \n",
      "Ep. 9100, Timestep 5860213,  Ep.Timesteps 1000, Score: 2569.05, Avg.Score: 2350.17, Max.Score: 2661.23, Time: 54:53:29 \n",
      "Ep. 9110, Timestep 5869030,  Ep.Timesteps 1000, Score: 2611.09, Avg.Score: 2317.83, Max.Score: 2661.23, Time: 54:56:55 \n",
      "Ep. 9120, Timestep 5876613,  Ep.Timesteps 1000, Score: 2594.65, Avg.Score: 2255.10, Max.Score: 2661.23, Time: 54:59:51 \n",
      "Ep. 9130, Timestep 5886613,  Ep.Timesteps 1000, Score: 2571.80, Avg.Score: 2329.68, Max.Score: 2640.66, Time: 55:03:44 \n",
      "Ep. 9140, Timestep 5896613,  Ep.Timesteps 1000, Score: 2651.05, Avg.Score: 2358.70, Max.Score: 2651.05, Time: 55:07:38 \n",
      "Ep. 9150, Timestep 5905685,  Ep.Timesteps 1000, Score: 2566.21, Avg.Score: 2342.25, Max.Score: 2655.22, Time: 55:11:08 \n",
      "Ep. 9160, Timestep 5915685,  Ep.Timesteps 1000, Score: 2609.11, Avg.Score: 2395.42, Max.Score: 2655.22, Time: 55:15:00 \n",
      "Ep. 9170, Timestep 5925685,  Ep.Timesteps 1000, Score: 2550.28, Avg.Score: 2396.64, Max.Score: 2655.22, Time: 55:18:52 \n",
      "Ep. 9180, Timestep 5935685,  Ep.Timesteps 1000, Score: 2594.05, Avg.Score: 2427.99, Max.Score: 2655.22, Time: 55:22:45 \n",
      "Ep. 9190, Timestep 5943873,  Ep.Timesteps 1000, Score: 2535.20, Avg.Score: 2404.12, Max.Score: 2688.39, Time: 55:25:56 \n",
      "Ep. 9200, Timestep 5953502,  Ep.Timesteps 1000, Score: 2539.87, Avg.Score: 2395.35, Max.Score: 2688.39, Time: 55:29:38 \n",
      "Ep. 9210, Timestep 5963502,  Ep.Timesteps 1000, Score: 2487.07, Avg.Score: 2424.92, Max.Score: 2688.39, Time: 55:33:29 \n",
      "Ep. 9220, Timestep 5973502,  Ep.Timesteps 1000, Score: 2660.51, Avg.Score: 2491.11, Max.Score: 2688.39, Time: 55:37:23 \n",
      "Ep. 9230, Timestep 5982560,  Ep.Timesteps 1000, Score: 2618.83, Avg.Score: 2468.31, Max.Score: 2688.39, Time: 55:40:53 \n",
      "Ep. 9240, Timestep 5992560,  Ep.Timesteps 1000, Score: 2618.47, Avg.Score: 2471.81, Max.Score: 2693.74, Time: 55:44:44 \n",
      "Ep. 9250, Timestep 6001960,  Ep.Timesteps 1000, Score: 2582.97, Avg.Score: 2482.66, Max.Score: 2693.74, Time: 55:48:22 \n",
      "Ep. 9260, Timestep 6011455,  Ep.Timesteps 1000, Score: 2614.04, Avg.Score: 2472.01, Max.Score: 2693.74, Time: 55:52:02 \n",
      "Ep. 9270, Timestep 6020715,  Ep.Timesteps 1000, Score: 2355.13, Avg.Score: 2454.94, Max.Score: 2693.74, Time: 55:55:41 \n",
      "Ep. 9280, Timestep 6030715,  Ep.Timesteps 1000, Score: 2585.20, Avg.Score: 2462.45, Max.Score: 2693.74, Time: 55:59:33 \n",
      "Ep. 9290, Timestep 6039913,  Ep.Timesteps 1000, Score: 2654.31, Avg.Score: 2488.92, Max.Score: 2693.74, Time: 56:03:05 \n",
      "Ep. 9300, Timestep 6048261,  Ep.Timesteps 1000, Score: 2640.45, Avg.Score: 2460.43, Max.Score: 2693.74, Time: 56:06:19 \n",
      "Ep. 9310, Timestep 6058261,  Ep.Timesteps 1000, Score: 2559.77, Avg.Score: 2468.53, Max.Score: 2693.74, Time: 56:10:13 \n",
      "Ep. 9320, Timestep 6068150,  Ep.Timesteps 1000, Score: 2645.44, Avg.Score: 2466.29, Max.Score: 2713.37, Time: 56:14:01 \n",
      "Ep. 9330, Timestep 6078150,  Ep.Timesteps 1000, Score: 2658.39, Avg.Score: 2495.09, Max.Score: 2713.37, Time: 56:17:53 \n",
      "Ep. 9340, Timestep 6087473,  Ep.Timesteps 1000, Score: 2673.19, Avg.Score: 2476.12, Max.Score: 2713.37, Time: 56:21:29 \n",
      "Ep. 9350, Timestep 6096706,  Ep.Timesteps 1000, Score: 2700.80, Avg.Score: 2469.53, Max.Score: 2713.37, Time: 56:25:05 \n",
      "Ep. 9360, Timestep 6106706,  Ep.Timesteps 1000, Score: 2654.38, Avg.Score: 2482.58, Max.Score: 2713.37, Time: 56:28:56 \n",
      "Ep. 9361, Timestep 6107706,  Ep.Timesteps 1000, Score: 2628.03, Avg.Score: 2502.67, Max.Score: 2713.37, Time: 56:29:19 \n",
      "Environment solved with Average Score:  2502.6711570321054\n"
     ]
    }
   ],
   "source": [
    "# Twin Delayed Deep Deterministic (TD3) policy gradient algorithm\n",
    "def twin_ddd_train(n_episodes=15000, save_every=10, print_env=10):\n",
    "\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_array = []\n",
    "    avg_scores_array = []    \n",
    "\n",
    "    time_start = time.time()                    # Init start time\n",
    "    replay_buf = ReplayBuffer()                 # Init ReplayBuffer\n",
    "    \n",
    "    timestep_after_last_save = 0\n",
    "    total_timesteps = 0\n",
    "    \n",
    "    low = env.action_space.low\n",
    "    high = env.action_space.high\n",
    "    \n",
    "    print('Low in action space: ', low, ', High: ', high, ', Action_dim: ', action_dim)\n",
    "            \n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        \n",
    "        timestep = 0\n",
    "        total_reward = 0\n",
    "        \n",
    "        # Reset environment\n",
    "        state = env.reset()\n",
    "        done = False\n",
    "        \n",
    "        while True:\n",
    "            \n",
    "            # Select action randomly or according to policy\n",
    "            if total_timesteps < start_timestep:\n",
    "                action = env.action_space.sample()\n",
    "            else:\n",
    "                action = agent.select_action(np.array(state))\n",
    "                if std_noise != 0: \n",
    "                    shift_action = np.random.normal(0, std_noise, size=action_dim)\n",
    "                    action = (action + shift_action).clip(low, high)\n",
    "            \n",
    "            # Perform action\n",
    "            new_state, reward, done, _ = env.step(action) \n",
    "            done_bool = 0 if timestep + 1 == env._max_episode_steps else float(done)\n",
    "            total_reward += reward                          # full episode reward\n",
    "\n",
    "            # Store every timestep in replay buffer\n",
    "            replay_buf.add((state, new_state, action, reward, done_bool))\n",
    "            state = new_state\n",
    "\n",
    "            timestep += 1     \n",
    "            total_timesteps += 1\n",
    "            timestep_after_last_save += 1\n",
    "\n",
    "            if done:                                       # done ?\n",
    "                break                                      # save score\n",
    "\n",
    "        scores_deque.append(total_reward)\n",
    "        scores_array.append(total_reward)\n",
    "\n",
    "        avg_score = np.mean(scores_deque)\n",
    "        avg_scores_array.append(avg_score)\n",
    "\n",
    "        max_score = np.max(scores_deque)\n",
    "\n",
    "        # train_by_episode(time_start, i_episode) \n",
    "        s = (int)(time.time() - time_start)\n",
    "        if i_episode % print_env == 0 or (len(scores_deque) == 100 and avg_score > threshold):\n",
    "            print('Ep. {}, Timestep {},  Ep.Timesteps {}, Score: {:.2f}, Avg.Score: {:.2f}, Max.Score: {:.2f}, Time: {:02}:{:02}:{:02} '\\\n",
    "                .format(i_episode, total_timesteps, timestep, \\\n",
    "                        total_reward, avg_score, max_score, s//3600, s%3600//60, s%60))     \n",
    "\n",
    "        agent.train(replay_buf, timestep)\n",
    "\n",
    "        # Save episode if more than save_every=5000 timesteps\n",
    "        if timestep_after_last_save >= save_every and i_episode > 0:\n",
    "\n",
    "            timestep_after_last_save %= save_every            \n",
    "            save(agent, 'chpnt_interm', 'dir_Walker2D_002')  \n",
    "        \n",
    "        if len(scores_deque) == 100 and avg_score >= threshold:\n",
    "            print('Environment solved with Average Score: ',  avg_score )\n",
    "            break \n",
    "\n",
    "    return scores_array, avg_scores_array\n",
    "\n",
    "scores, avg_scores = twin_ddd_train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(agent, 'chpnt_ts2500', 'dir_Walker2D_002')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of scores:  9361 , len of avg_scores:  9361\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAAEGCAYAAACpaqUaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOydeXxU1fn/32fWJIR9F5QoEiAsgiIuRcW2uNddW+3PpbXVSq3V2sXW3Wprq9ZWv8W1CqJWK+64g8giouz7DglbgLAlgSSznt8fd5k7M3dm7iQzJIHzfr14MXPuufeemUlynnmez/M8QkqJQqFQKBQKRUvH1dwLUCgUCoVCoXCCMloUCoVCoVC0CpTRolAoFAqFolWgjBaFQqFQKBStAmW0KBQKhUKhaBV4mnsB+aBLly6ypKSkuZehUCgUrYr58+fvklJ2be51KBSpOCSNlpKSEubNm9fcy1AoFIpWhRCiornXoFCkQ4WHFAqFQqFQtAqU0aJQKBQKhaJVoIwWhUKhUCgUrYJDUtOiUCgUipbD/Pnzu3k8nheAwagvy4rURIFl4XD4ZyeccMJOuwnKaFEoFApFXvF4PC/06NFjYNeuXfe6XC7V8E5hSzQaFVVVVWXbt29/AbjQbo6yeBUKhUKRbwZ37dq1RhksinS4XC7ZtWvXajSPnP2cg7gehUKhUByeuJTBonCC/nOS0jZRRotCoVC0ciJRyf/mbqY+GEFKZRsoDl2U0aJQKBStnFfmVPD7t5Yw8N5PeOSTVc29nBbLH/7whx7HHnvsoNLS0rIBAwaUffHFF22ae02K7FBCXIVCoWgBTF9TxXUvfsvsO7/LER0K087dvKeOV76p4D8zN3JEh0I27akzjz07fQN/PHdgvpfb6pgyZUqbTz/9tMPSpUtXFBYWysrKSk8gEBCNvV4oFMLr9eZyiQoHKE+LQqFQtAD++80mABZv3pdx7i2vLeDZ6RsIR2WcwaJIzdatW72dOnUKFxYWSoCePXuGS0pKQtOnTy8aPnz4gP79+5cNGTJk4N69e111dXXi8ssvLyktLS0bOHBg2QcffNAW4Mknn+x87rnnHvPd73732NNOO60U4J577uk+ePDggaWlpWW33377Ec35Gg8HlKdFoVAoWiCrttewYlsN3dsVMGXlDu77wSAA9h4IUlnd0Myrazy/m7T4yDXba4tyec3SHm3rHr38uM3p5lx88cU1f/3rX48oKSkZPGrUqJqrrrpqz/e+970DP/7xj/u++uqr688444y6PXv2uIqLi6MPPfRQd4A1a9asWLhwYcF5553Xb/369csAFixYULxkyZLl3bt3j7z99tvt1q1bV7BkyZKVUkq+//3vH/vxxx8Xn3vuuftz+foUMZTRolAoFC2AT5Zvj3t+zj9nxj2/9bv98LgFw//8+cFc1iFD+/bto8uWLVvxySeftJ06dWrb6667ru/tt99e2a1bt9AZZ5xRB9CpU6cowOzZs4t/9atf7QQYPnx4wxFHHBFcunRpAcBpp51W07179wjAJ5980m7GjBntysrKygDq6upcq1atKlBGS/5QRotCoVA0E9e9+C0dirz860fDM849VIyVTB6RfOLxeLjgggtqL7jggtqhQ4fWP/PMM12FEEnpVukysIqKiqLWebfddlvl7373u115WrIiAaVpUSgUimZi+poq3lu0jZ01sXDPza8uoCEUacZVHZosXrzYv3TpUr/xfOHChYX9+vVr2LFjh2/69OlFAHv37nWFQiFGjRq1/5VXXukEsGTJEn9lZaVv6NChSTG5c889t2bixIldqqurXQAbN270bt26VTkD8oh6cxUKhaIZWLGtxnw88i9T449V1iROVzSRmpoa96233npUTU2N2+12y5KSksCECRMq1qxZs+vWW289qqGhwVVQUBCdMWPGmt///vc7r7nmmj6lpaVlbrebZ599ttwQ8Fq59NJLa5YvX15w4oknDgDNC/Pqq69u7NWrV/jgv8LDA3EoFiIaMWKEnDdvXnMvQ6FQKFJScueHKY/dff5AHvpwZaOvXf7I+Y06TwgxX0o5otE3TsHixYvLjzvuOBVCUThi8eLFXY477rgSu2MqPKRQKBQtjKYYLArFoYwyWhQKhSKPXDruK47900dsr26g5M4P+WDxtuZekkLRalGaFoVCoWgCldX1RKKS3h3tS48s2KQVi7v51fkATJhdzvZWXGdFoWhOlNGiUCgUDvn3tHV8f2B3+vdoC8TrUqw6kutf+pZlW6uZd/cYc2yhbrzMq9jLvIq9B2nFCsWhRd7CQ0KII4UQ04QQK4UQy4UQv9bH7xdCbBVCLNL/nWc5549CiHVCiNVCiLMt4+foY+uEEHfma80KhUKRinAkyqOfrubif3+Vce6Xq6vYtT9Ixe4DB2FlCsXhQz49LWHgDinlAiFEW2C+EMKojvSElPIx62QhRBnwI2AQcAQwRQhRqh/+NzAG2ALMFUK8L6Vckce1KxQKRRxGnmV9KML7i7cxYXZ53PH/fruJV+ZU8M7Y75hjZzz65UFbH8BgsQE30cwTFYpWSt48LVLKSinlAv1xLbAS6JXmlIuA16WUASnlRmAdMFL/t05KuUFKGQRe1+cqFArFQcNaHeLW/y5kfkKI549vL2X5tpqDplfpQC3/z/057ThAecHVLPTfyGT/3fzF+5+Dcv/WyMsvv9xBCHHCwoULC5p7LQDbt293n3TSSaVFRUXDr7322qOsx2bOnFlUWlpadtRRRw2+/vrrj4xGNWN0x44d7lNPPbVfnz59Bp966qn9qqqq3LlYyxlnnHHsrl27mnStyZMntz3zzDOPzcV6UnFQsoeEECXAcOAbfegWIcQSIcSLQoiO+lgvwFreeYs+lmo88R43CiHmCSHmVVVV5fgVKBSKw5Ex/5jOjS9rNZ/21QcdnSNEPlcU40bPhzzkfYklBT8HoKPQ2t0MclUcnAW0Ql5//fVOxx9//P6JEyd2au61ABQVFckHH3xw2/33378l8djYsWP7jBs3rqK8vHzZhg0bCiZNmtQO4L777us5evTo2oqKimWjR4+uvffee3vkYi3Tp09f16VLlxZfijnvRosQohh4C7hNSlkDPA30BYYBlcDjxlSb02Wa8fgBKZ+TUo6QUo7o2rVrTtauUCgOb9bu3M9nK3bwg6dmcd97yx2d88bcg9NapwOqJ182VFdXu+bNm1f80ksvlb/zzjvGl2XOP//8Y9544432xvPLLrusZPz48R1qa2td55133jGlpaVl559//jFDhw4dMGPGjKQUsffee6/twIEDy0pLS8uuuOKKkvr6egHQq1evIbfffvsRZWVlA0tLS8vsvDvt2rWLnn322fsLCgriYnoVFRXe/fv3u77//e8fcLlc/PjHP9797rvvdgT45JNPOtx00027AW666abdH3/8ccfE64bDYW666abegwcPHlhaWlr26KOPdgHNEzJixIj+Y8aM6du3b99BV1999VGRiGan9OrVa0hlZaWnpqbGNXr06GP79+9f1q9fv0HPP/98x3Svc9KkSe2OPvroQSeccEL/SZMmdTDWUFNT47riiitKBg8ePHDgwIFlr7zySgeAefPmFQwZMmTggAEDykpLS8usrRWckNfsISGEF81geVVK+TaAlHKH5fjzwGT96RbgSMvpvQGjoEGqcYVCoWgym3bXMXXVDn7ynaNtjy/dWs3SrdWOrlV5kMJDDfhsx68P/p7xB2UFjeTdXx7JzhX2+eGNpVtZHRf/O621+Oqrr3YYPXp09dChQwMdOnSIzJo1q2jUqFF1P/zhD/e88cYbHX/4wx9WNzQ0iK+++qrdhAkTKv7+979369ChQ2TNmjUr5s6dW3DKKacMSrxmXV2duOmmm47+7LPPVg8dOjRwySWXlDz66KNd77333p0AXbp0Ca9YsWLlI4880vWRRx7p/sYbbzhyg1VUVHh79uwZMp736dMnWFlZ6QXYvXu3p0+fPiF9PLRnz56kffyf//xnl/bt20eWLVu2sr6+Xpx44okDfvCDH9QALF26tM3ChQuXlZaWBk8//fR+L7/8csef/OQnZqzz7bffbtejR4/Ql19+uU6/nzvV6/ztb39bdcstt5R8/vnnqwcNGhS44IILjjGu86c//annmWeeWfPmm2+W79q1yz1ixIiBF154Yc1TTz3VdezYsTtuvvnmPQ0NDSIczq7jQT6zhwTwH2CllPIflvGelmmXAMv0x+8DPxJC+IUQRwP9gG+BuUA/IcTRQggfmlj3/XytW6FQHDqEI1H21aUP60SjktMfncYDH6xgzobdTb7nWwuSPP15wUuY3bItW2VnKmUnLg3cT0nDa8wic8fow5H//e9/na666qq9AJdddtkeI0R0+eWXV8+ePbtdfX29mDRpUvuRI0fWFhcXy9mzZxdfddVVewBOPPHEhtLS0rrEay5evLigd+/egaFDhwYArr/++t2zZs1qaxy/+uqr9wKMHDmybvPmzY49CnbtdUQWcccpU6a0+9///td5wIABZcOHDx+4d+9ez4oVKwoAhgwZcqCsrCzo8Xi48sor98ycObPYeu7xxx9fP3PmzHY333xzr08++aS4c+fOkVSvc9GiRQW9e/cODBkyJGB4hIzrfPnll+2eeOKJngMGDCgbNWpU/0AgINatW+c75ZRTDjz++OM977rrrh5r1671FRcXZ9VLKJ+elu8A1wBLhRCL9LE/AVcJIYahhXjKgZsApJTLhRD/A1agZR79UkoZARBC3AJ8CriBF6WUzvy0CoXisOaPby/lzflbWPfwuXjc9t/R7ns/9ufkR8/N4dpT+nDvBWVMX3PwtHElopKdsiN1ONOHXuP+jGs8U9giuzAq8GTcsYE92+Vjibkjg0ckH2zfvt09Z86cdmvWrCm85ZZbiEQiQgghn3766S1FRUXy5JNPrn377bfbvfHGGx0NQ8VJX75McwoKCiSAx+OR4XDYsdVRUlISMjwrABUVFb4ePXqEADp37hyuqKjw9unTJ1RRUeHt1KlTkqtCSikef/zxTZdddllc583Jkye3TTR+Ep8PHTo0sGDBghVvvfVW+7vuuqvXlClTai699NJ9qdaaypiSUjJp0qR1xx13XMA6fvzxxzecdtppB95555325557bum4cePKL7zwwtqUb0YC+cwemiWlFFLKoVLKYfq/j6SU10gph+jjF0opKy3nPCyl7Cul7C+l/Ngy/pGUslQ/9nC+1qxQKA4t3lukRZLD0dSby8Q58R77l7+uYNa6Xcxen97r4iZCEdmFgsa45nGsSPbEfOm/gxUFP3V8nT97xwPQEcd/6w9rJk6c2PHSSy/dvW3btqVbt25dun379iW9e/cOfvbZZ8UAP/rRj/aMHz++y9y5c9teeumlNQCnnnrq/tdff70jwPz58wvWrFlTmHjdYcOGNWzdutW3bNkyP8DLL7/c+bTTTmvyh9KnT59QmzZtolOnTm0TjUZ59dVXO1900UX7AM4+++x9zz77bGeAZ599tvM555yTZFCMGTOm+umnn+4aCAQEwJIlS/w1NTUu0MJDq1at8kUiESZNmtQpcb3l5eXetm3bRseOHbvntttu27Fo0aKiVK9z2LBhDVu2bPEtX77cD5rQ2bjOmWeeWfP44493N7Kevvrqq0KAFStW+AYOHBi4++67d5511ln7Fi1alPS+pkP1HlIoFC2WhlCEbfvqG32+8SXwn1PWsmq79qVzwuxyFm3ex5a9dSk7LbuE4MvVO9Ne+zHvM1kZGiB53vcPpvh/n3JGKiOoM9WUF1zNha7Z+DClDrQRAdv5injefPPNzpdeemlcjvpFF1201wgRXXLJJTVz585tO2rUqBrDO/K73/2uavfu3Z7S0tKyhx9+uEf//v3rO3bsGJddU1RUJJ955pnyK664om9paWmZy+Xit7/9bVYuul69eg255557jpw0aVLn7t27D50/f34BwLhx4yp+8YtflPTp02dwSUlJ4IorrqgGeOCBByqnTZvWrk+fPoOnTZvW7oEHHqhMvObtt9++a8CAAQ1DhgwZ2K9fv0E///nP+4RCIQEwbNiw/XfccUfv0tLSQUcddVTgmmuuiTN65s+fXzhs2LCBAwYMKPvb3/7W8957761M9TqLiorkU089VXHBBRcce8IJJ/Q/8sgjzVjsI488si0cDosBAwaU9evXb9Ddd9/dC2DixImdSktLBw0YMKBs7dq1BYao2CnCiQustTFixAg5b9685l6GQpEXahtCrNmxnxP6JCUNHHL85KVvmba6io1/PS+rmL7BgHs+piEUS8wof+T8lIaKlVduOInb3ljIrv32epiBooKP/X8EYFjDs+yjre08A0GUb/1j6So0w6mk4TXzWDsOmGnLACManmYX7ePOv9g1i3/6xlEjCxkbuo1XfH8F4PPICfw8dEfc3CG92vPBr0ZlfI226xRivpRyRKNOTsPixYvLjzvuuF25vm4+CYfDBINBUVRUJJcvX+4/66yzStevX7/MMGpaI5MnT277+OOPd582bdq65l5LOhYvXtzluOOOK7E7pnoPKRStjBMfnkJDKMqKB8+myHdo/wpPW619aQ1FJD5PdkbL7v2BOIMlG1wu6NzGn8JokabBAnC2ex5vRM5Me73Botw0WBKxGiwAl7ln8GzkB3FjF7u11gHtRD1d0LKYfhi4h2/kwEwvRdFIamtrXaeddlr/UCgkpJQ88cQTFa3ZYDlUUOEhhaIVsXlPnbkRh6OSUCTKPz5bzcpK+w2xtWM4V4KR7I0PO03K1JU7bGYm4xKCMwd0Sxr3E+S/3oeTxjLRXcRXzy0mKRGF+0PXAvBzT7InqFLGaqH90zcOgA2yZ9I8Re7o2LFjdNmyZStXr169Ys2aNSuuvPLKVv9LdsEFF9S2dC9LJpTRolC0IvZa0nf//cU6+t31MU9+sY5z/zWzGVeVe+ZX7KHkzg/N0vnRHIWx/8/h32uXELb3HO1azCnu+LZnPrTkjZ7sphP2+9oLPq2G5tJoCQDXuj8D4g2e8RGtR2zQxgHeViQbOVUJIaQWTjQajR6kWsGK1oz+c5LyW4oyWhSKFkxDKMKf3llKMBxlZWUNj3662jz27IwNzbiy/BCJSq56bg6XPf113LhsRJTHTgKzcZezrsvbaxpYtT0xCUTyrO8JAM4I/IMhDS8AcKvnHQC+LvgVCwp+kfa6r0TGAPB77/8YIDYx3vt3AKZHhgKCedFSjhB7KBExbeUN7g+5wP2NzdXsbYCD1UYgS5ZVVVW1V4aLIh3RaFRUVVW1J1a/LYlDOyCuULRC/vbJKjoUernshN488MEKPli8jde+2ZSz689au4v/9x9tEyx/5HwA5pXvoWJ3HZed0NvRNaSUvDKngguGHkHHNvaVWRvDm/M287VNgbcbJ87jjZtOSbue+lAkTuMjbDb1fXWhpDE73l24lRkJdVqGi5iXpkL2wOgm0k7U8f/cn5OOOumnUnZiWmQY6NU3PvHfaR6fFh0GwAjXGkBLgTbEuvd4X0263rXBPzh6HS2FcDj8s+3bt7+wffv2wagvy4rURIFl4XD4Z6kmKKNFoWhhPP3legD++vGqvFzfMFgAVm+vpX+Ptlz+jObZcGq0LN9Wwz3vLWfyksq0xkS27E1hVHyzcU/a856dsYFHPl7F3Lu+T9e2frbsreOXry3I+v5tqeN89xx2ySuTjv1K96jMi5bqI4KXwmfzE8+nPOR9Ke11d8l2LJZ92UlHShpepbzgx3HHF0a1xrh7ZbHZ+DCR0YHH+dKvZQptlKl75LVEV8YJJ5ywE7iwudehaP0oi1ehyANfrt7JNxlKwv9swlzuf795izvvPpC6zseGqv3srLGvGxIIa/GaTMZEtvztk+wMtXnlmvblhZkbAais1mq6jPrbtEbdf6znPR7xvkD/wNK48YGigu+6tcLe/y8YyxzaIZPTzr/vmp805hchAtIocBpvVjwQuobFUjNahgeeA6BWxuptzYwMBqDcIrwNSfV9U3F4oowWhSIPXP/SXH743Jy0c6as3Mn42eWAVnvlmenriaap3JqJdDWXDgTCfGtjYFz9/De89NVG87n1/t99fDoj/zLV9nq51k3s2t+4ImlGNVvj/HSVb53QS2ilRLpE4rOM+lmq2DYQayHzefSEpGsYolsrfkIEMKuysyEa85RUyO5xcyeEx1BgEejWUcDK6FFxc0LKSa44TFE/+QpFC+DhD1fy+tzN9O1anHlyCtZXHeDYbvbnn/XEDLbuq+d3Z/dPOvbAB7FsmG/L93DyMZ0bvYZseWjyCuZW7GXx5n08e02yAZCJRDst0gSjpZg6LnTrAmAR/33OJ7QMobXRXnHj62Uv+jZMJIKb8oKrzfGu7KWKmBcm0Wi5MfQbszJuotGyl7Z4RQRBFIkr6VxQRovi8EV5WhSKPFJy54dMXrItbiwcicZ5RXbtD1DboG2KDaG4KuFZYZSpT6SmIcRWvRS+NfvIDmNZdcHs2sU3lhdmbWTxZq2K+E0Tk8Mq2dKUkv/Hu9aaj8t3Vscd8+ppzdfZCGAjuAGokrEU5COFVcQrKRRBGogJltfJmPGzX8a3XmmQ2jy/Xq6/wHLuu5FTAajDccNgheKQQhktCkWeueW1hebjumCYY+/6mCenxjJRRjw0hQ+Xaimut76+MOl8p9g5GYLhKEPv/8zxNVx62Gfi1xXpJ7ZQfv36osyTUjDCFTPouhe7444ZRks9qTOlTgyM46bg7YCmYTEw6rjENC1g1bXUUhR3nXrdIDFCRH5iepg/hn7GkIYXCCtPi+IwRRktCsVBpKZe28CemLLG9nhTaqjZaVpCWVaSdetWS2V1+u7FVklL/7s/ZtnW6pRzG4dkgMic5m33dm3ek1yILRO9RRW3et41n/td8e+bYbSkD8sIU5jrtzQ1NIyPxBDPk+GLWRI9mjoK4sb7Cs0zN8S10TzfOLeegiQjR6E4nFBGi0KRBVJKXpy1kZ21qTf1dGLaDVX26axO6cZefu95nSvcXyYds+uT48pSMevSjRZDIJwKa92YQDjKCzNzW+juOvdnfOK/kxEi+7Tv0/6efebQeO/f4p57ZHx4zIcWtsukJTGMC2ul2zZ65+b9CcbGP8JXcmEwviUAwLUerebLRN8j+r1DcaElheJwRhktCoUNdcEw47/aGOe9WF+1n+lrqnhw8gpGPjyVdTsTK6ZqRGw8HoZA9OoX7CqbZqa3qKK84Gq+LfglYz3v86j3uaQ5O2zSk9dnaSSt27k/bRYSwEdLK3lz/pa4sXcXbUsxu3GMdK0EYJL/QbNBoB256FJ/pNjBsS5t/UZ6sYd4bZEzTwumcWH1tBgl+GsTtCupWBI9GoADUgsTxadLKxSHNyowqlDY8OAHK3h97mbu/2AFPz/taO46v4zvPT49bs64L9fz8dLtFPrcLLhnjDn+8Icrk643r3yP6cVoDCeJ+GvWyZgQ88+eF+kh9rKmMNmQueCpWVnd5/eTllDsT/9nwS51OtcMFbE07HkFNzO04fm83esFr5ai/OvgWD6Knsxa97W4EzwtHhEmIgXRDN/zDOPCqmkZ5dIqku/HmdEyNvRrZvlv469hLRupgKDytCgUOsrTolDY8Prczebj52duJBBOzup5e8FW6kMR9hwIxmXb2IVWlm+rYX7F3qRxp9QkhBaKRIDOugfiGs8Uxrjn064gN99BUhWUMzAynfLJFtk17nlPYV+oLxdtFPu7NK/RcllCSM8E8pAcHnKSZmyEh6x1Vi50fwXABnmEo/XUSU3j4tJ7xrWlngMJupe0tNDmQwpFLlBGi+KwZ+mW6oxhhoZgekFrIKQdn7pyh+3xJ6asaVINESM8MSbwd54La/2C+gj7e+WbtxZsyTypCQii9Hdt4ovIMHPsOvenebpb7DPR0pAFIelO8rR4Cdt2X04kpmmJeVoqZWc2R7smGWKpMLwqBQRxEcUvQqYho1Ac7iijRXFYM/rRafzg/2bx3283p523NUP9D4nmobhhwjzb4y4hbGukFFPHeO/f4rr6GpSKzZQXXE15wdWM8z0JQBg3UyLHA/Bjz9S4DBtPOPusmcZwal/74nNNqTFj5SzXfDqJ/XwcHclfQlcBcLIrOeQG0BBs2j0Nw++h0I8xcqLCuHGTbLQ487Qka1q6iGrHBot2jZjhY3hsGlCaFoUClNGiOESobQjx6jcVjoSZ0ajks+XbkVJSvlvb6P/0jtZrZl9dkPGWsvYGiQXi7JixdlfKY6k89ie41jLavdhshGflM398IbN50VI2yW5mYbHL3DPjOgUXBHOjNanMEB6qT2GclO8+0OR79xHbedb3BABfRQbzXOQHzIgMSQqPGUxdtbPR93IRZZjeuXm9JXQTwo1bJgtxnRgtRnjJqKAL0IVqdtHO8boiuAlJNwUiaBot9aqYnEIBKCGu4hBgfyDMH95awkdLt9OvW1tGHt0p7fwJX5fHla43GP/VRu63GQf7jCArUko8WQptXUR5wvtv8/kV7i95MzI65fxrgncSwZ1yA/NEcuNpMWrJpGLhpn05uY8dHYhlO22jC6B5Hrw0zaPSjv2Mdi3m/eh3zLF/eMdxsXs2EG8UhHEna1pE2GGTQkFAesxwHkBnUcOuaPs05yTTgI8CQhQSMJ87RSlaFIcyytOiaPUMvu9TPlq6Hcgconhx1kamrrT/dp7KYAHYbVMDxcqybTVmYTY77I5c4Z5OJxHbpP/secl8fIP7Q/PxT4K/4+SGp6jXxZip9A0L1uZGa+JzN9+2J/V36oZgzPMUIt4IaAx/8b7Ik75/M9KShWUYLAAHLO9pGA9EG6dpMdbrM0rwE6CdqGe3dO5pAc1Q8xOkQOiF6aTKHlIoQBktilbC7v0BNu7KHH74aGll2hDRg5NXMGtd6jBOKibNT28QXPfit3jTbPZuV+KvmuRy93RWRY+kb8NEyqPdKbCkyZ7pipWjnx8tZTsxHck+2gAwJzqQF8LnmuPVVel1OU7xuhv3Z2HGmqrMkyzMr0gOZxnhEGuV2DDuJhste2RbAIa67IvgWXv5hHBTsz/ea+U0PKSdHzOybvO8BcBo9+Ks1tuAjwIRokA3fpSmRaHQUEaLolVw+l5z/4IAACAASURBVN+nceZjX2ac9/rczdw0cX5S88C73lnKU1PXpjgrN+yrC6U8VuiL/1V7w/dnTnSt4e3IKCK42SS7xR3fRSyckBgaqKeAHwQe4mfBO3g4/GMuDjyo3SMaLxbOJlupkAbTO5BtPZkBYhN+gvzlo+yq106avzV5HaZnIbZJh/DgFU0LDxleEus7sls3ZCZHTqZc9ojdT3rwCjshbnw/onT3MowWQ5Cb2B06EwGpeVr8StOiUMShjBZFq+BAiiyR6vpkQ+GzFTs4558z48Ze/WYTj39u3+8nV9z59lLb8WPEtiRNxkkubYN/KaJ5SuZEywBMw8Eo/T4/2s82LLFUHsN+ipC4qJSahsdN/Hvxko2gOBUrC37Ky3rZ+GzoRA2f+O/keb1AWzZMXpwsbi7QNRzWTToomx4eMnATjXs8PnwWt4RuNTs1g6Fpif+8fFl6Wvy60WN8rq9ExqQ7JYmAoWkRuqZFhYcUCkAZLYpWhlHEbV9dkMrqeq585uuUcy8d9xX/meV8484H3djLF/7fckvoJcto7Pt+4kZ4kmslRTQwxr2AhdFjuSz4AJmklcY1EvvlPGRTmdcOQ3SaKq04Hd2EJso93W1vsKWjNpBsiMSyZWKbtJ0R4YTr3J9yjutbRrsWmlVpfRbjx0eYoE3YJYQ7ycj0OiwuB5qXxDCyjHUHssx5aNA1LYbxWpeFp0XVllMcyuQte0gIcSTwMtADiALPSSn/JYToBLwBlADlwJVSyr1CCAH8CzgPqAOul1Iu0K91HXC3fumHpJQT8rVuRcum7N5P+ftlQ/nzhysyVmZdsGkfCzbt44cnHnmQVpfMvd6JAAyPxDb1YrQwzmOhK8yxSZHT+b33DUa6VlElOwCwKups3cZm6pLpN/YxrnkMEJt4KnJp3Hh7Gp+qfIpreaPPtcMID9VLq8akcZ6WB7zJfyas5fV9hJI6L4O9keQVYeodejus63XasyiRgPThFyG6Cq3q8U79Z0KhONzJp6clDNwhpRwInAz8UghRBtwJTJVS9gOm6s8BzgX66f9uBJ4G0I2c+4CTgJHAfUKIjnlct6IZCUeizC3XBJorK2sIR5Ir0f7+rSVZlZIffF++qqlmpgNaU8VZ3lPNsU5CG9tOLDV7Jx0JSA8eIhSjiUA/jJ7s6B5B09OSWlMD8LzvH9zhnZQ0Ps73r7jnCzc5bzcwzLUegKoss2NSYZfia+f5yEQR9rVmTN0OUTwiStAmjTlsYyRlJ8R1mx4dQxvjLF06RgNeCggyRGjC4d1klzKtUByq5M1okVJWGp4SKWUtsBLoBVwEGF+BJgAX648vAl6WGnOADkKInsDZwOdSyj1Syr3A58A5+Vq3onm5YcI8rnjma96Yu4lz/zWTEx+e4ihrqKXSUU9pttpY7XTPxj5ZzPWnlpjjUVx0oZr2Qjt+wGHpdmMzdeqN+IPnv3HPDX0NQBvqmVvu3GgxNn1fjjQnhWYFWKvRkr2nZan/hrjny6Il7JXFpjDWMF5ShYeSuzw7Dw8FiYWHfI30tPQUezjOtYGjXVoqf6ZGjQrF4cJB+U0QQpQAw4FvgO5SykrQDBvASJvoBVhzNrfoY6nGE+9xoxBinhBiXlVVdqmXipbDdD1tdtFmTSuxty7kKGuoORggNvG89zGzsV0iw8VaBrkqAAgEYpk9xkYWwMvZg2JZK4UiyBWeGVzo1nQ6TkMCUVxEpEjKeEnFzZ4PUh7rIbKrqmuEWwwPSVPxiyBRKeLCNmHceEQUoun7PxkcJXbgFvGZUz8K3k0Ar2ms+E2jxcbTIt14ROMq4oIhxA2Z52ljzjKPDL7WBbwnuVYxJzowq3MVikOZvBstQohi4C3gNillTbqpNmMyzXj8gJTPSSlHSClHdO3qvM+H4uCwcdcBJth0PwZ4aPIKPl8R3/wvUy+glsBD3hcZ414Q1//HoAvVvOO/z3xu9RQYG1pQeuNEk4+HLgfgIr3o2a4sQgJhPPiyCKEcbdPrSFtndmEY03MhIklVZBtDIUFdhBt7Y8wQTjR9+MvgdNeSpLH9FBGUMWMivafFk+Q5yqa4nDXbqbGalkfCV2U1X6E4XMir0SKE8KIZLK9KKd/Wh3foYR/0/43ypFsAq/KwN7AtzbiilbC+aj9nPzGD+95fTshGo/LCrI38/GX7RoMtGSN9dpCrPOnYcFd8TRjrJpjqW/6LkVihOFncw2y+54RgliGUaZZeRxXRbtRIrbdPtmEYa2PANil0JNlQSCCpLo254UfSVyU22KvXX/lnOF5wHMBnrtdISbYzROzCQz4RIiyzr9PSaCGuxZhqilBaoTjUyJvRomcD/QdYKaX8h+XQ+8B1+uPrgPcs49cKjZOBaj189ClwlhCioy7APUsfU7QCXpi5ge89Pp2gbqzMXr875dy9B5xtSi0FY6O3I1EIat3cjW/5iZkrByg0H0fPuJNsCDWiamwbPYupUASplm3i1uYU6/xchIg6iRpzLQZhI7QScba2i9xfAbAoeiwAK6NHAZox4Te1JvpnIO2yhzxJRksbGhynHVs9NYbnKtvwkNXTdEvoV1meq1AcuuTT0/Id4Brgu0KIRfq/84BHgDFCiLXAGP05wEfABmAd8DwwFkBKuQf4MzBX//egPqZoBSTWCrnuxW/jnltL7n++Mj5E1NIxjA47D4NLj2DOiAxhi+wSZ1BYNS2pSmrIsotTHLHHiVh1jyyOe96WOn7j+R/dxD6q9dYAPoe6GANrCnGiDqQxdBXV7JDxDS9jnhZnRstK2QfQ2hzcF7qOa4Nat2yjnw/EPgO78JBd24DGCnG9IkxIupFN+FO7XmZXTVehOJTJW50WKeUsUlfF+p7NfAn8MsW1XgRezN3qFE1lZ20Db87bwtjRfRG6MGPFthqiUjK4l6bF2G9TPCwRq+elvJVlCUX0jciou2KlWGhjt4fGMsl3f5wx0MYdM1oSuSTwAGe75/GzguxSXEN4kgyOvl3bsL5Ke0+7UE0nsZ8J4TEcLbZzunspnUQtt3reBTC9G/4sPS3xHqSma1rac4CN9IwbM40Fh5qWEqE3z8THhMjZ5ngQr/kepfJ2afdLDg95CMc8PhmwfhbZCHgTuTr4JzrpKfPZoGrLKQ5lVB6dolHc/sYiHv10NUu3Vptj5z05kwuemsVpf/8CcFYfpbYhthGN+3J97heaQFf2Mtt/C31Fct+bbDE2traiLulYB7RU51qKtM3SGkZxaecFbUITC2W/Rokw7UrdW9XqN3omA9BFVDNe38i/71pgHjeaMGYbHvITIiq1bTIXpfbbibqk8FDI0JI41LRcZHZvjt++tX4+iUJcJ9lDEp+IOO89JGPhIV8WPYsSmR0dzOToKY06V6E4VFFGi6JR7NcLjxg9+azN+TbvqefPk1c4uk4o4rypXy4Y63mfI8Qepvp/1+RN1tBwXO6ekXSsm9jHXllMEC/BhGwUI0QRxGN6qZpKyEaHYW12PT9aCsC48MVmB+WjXDvN40OE1u4g2+whHyH269fLhdHSngPUEK8VCpvhIWfXr4h2sx2PS3m2ZHAlkljMzhBcOy0QlyjEbaynpbHk6mdKoWiJHNzfJkWrp7YhxJa99ea3+HHT1vHZimQtitOePwe7cFwvsct8PMK1mq+jgxp9rQ56EbjOopZjxRbWyd7msW5iHzukVrg5UW8Sp2nJ0f5ip2mJWqyWWDjEg0svWteNWBG5o1xVcfOcUiCC7KeQdtQ3OTxURANFIpDkaQlmmT20i/ZsiiQbLkGsnpZ02UPx76VhDIYd/rmML+PvXAujUCgyozwtiqy4YcI8zv3XTHNDtDNYsuEfOe+8LClMk3q7WcY2s/94H2vSnbrqzQIh5qkw6Cb2mcXhgnjjBKveNDVCGotdbRGrp8UqPDWyYKxNDn8U1Fp7ZSvELSRIrZ5F1ZimhlYu0z1WG2W8piWWPZTZaOlCNSe41sZ1bTawelrS1WlJzB7KtkCc5lnTi8vpQlyFQpEblNGiyIp5el+gcA7COp8u397kayQy0ftXVhb8NGUjv7Pdc83HRaLxKbqCKN0tRkuioLOr2EeVXhzOqnEATQcSkcJ2Y20sdnVarJ4Wo1puUHqS2gOUNLzG2qiWoZK9piVIrR7OcVqRNxVGPZIp0ePjxmNC3MzX/9J/OwCj3YuTjgUsxeX8aT0t7hSelmyEuBFANkt4SKE4lFFGi8KWs56YzgszNySNe93aj0zF7mTxaba8OS/3VW9Pcy8D4L++h22P99bDQ19FBrHSYRdlO/rpQt6pkeEAFBDzAgiidKXa7NacaFBo3YV9+tzcEJbuJKMh3tMS8ywYmhaABXotE8PjkI3R4iKKX4SplVp9maZqWnwilFTCX1ub8/BQsUjtZbMWl/OJ1NlDiV2evVkaLYZOxkcYLxHHlXQVCkVmlNGisGXNjv089OFKJi/Zxtd6WnJldT2BsCZKrA81vSbHtNW57RHltxgOsyNltnMWRI9lTnQgdRQw0NU4o6kL1Xzm12p/TI0eT1QKs7EdQFeq8YsQW6TWTiIxe8hHKOcbWaY6LVYNh7VI2qeRE81xbZ7zz9V4v/frRfGaqmkpFVtxieTOHWYlWgd1WoxicXb9eqyCaFNXZFtcTu91pCu3PGaBOGefWUCfV0BQeVoUihyjjBZFHNv21fPM9Fjq8S2vLeSq5+cA8IuJ83N6L2vGUbZ85PsjU313xI3Z9QBKxEOEOunnGGF0gsh+Dd1ETMC6UfZgiTyG40WsbP/1nk/1K2ubb3L2UCinehYwNC3a5rprf4AHPlhO2NJg0OppsQpKjVcfM1rSGwbWYoCG1yJXmpZzLKE7K9kUl1sk+wLwm+DNScfii8ul6T0kja7Z2uvx6B4sp2X8jTYCHcR+ZbQoFDlGGS2KOE595Ase+XhV0ngoEmXxlmqbM5qHMlcFfV2VCEuH5ZNdWvXdallEUYowgbGJvBU5HYhtvIv9P+Nd391J5fftsJarnxvtT60sjBPa/sT9CYDpaQlJT1zoxkvI/Iafq+yhoEWHce97y3jpq3J21MTWmarXTlehfaYSFyHpNsMmqXhy6jrzsXE/w9OSi5RnO0wBrIPici6izIoMYhtdko4FpRe3kLiJpK/Tot/PY8kAiltHBnbrWqYZ/tvxi5DjsFKuUAnPikMZZbQoHPH0QSj85pzYt32rlsQwVNqLOoa5kvU4oIUFgnj0TsKx841zXvM9FDd/jGse5QVXc7yIZTkZAt6fBn9LGE9S+GeN7E1ECqZHjwOSw0P+uPBQ7uq0GEaDnUjaR4iA9Jj3ezr8AwCmRYeZcxI9Qna8vzhWlM8whIxwU7aZR1YM4zOx1QBk1zDRb9ELJWLoV/yELE0r7bs8Q8wIy1aIu1v3tACc5FoV61J9kFBlWhSHMspoOcx5a/4WbpqYucNy7lOTG4+1dLxhdIwQqxgu1lEn/VTr4YrOJHuGvKbRom20hQRwWbw1icbO8z6t1+fb/vvNsSLd01IpOwPxm30b6jnOtYH18ghzvrVRn7aGPISHpCdt7x/NWIvd82/hqxja8Dyzo4Mt6/SmDQ9d/vRssy0AxDZ1Ixsp28J0VozP8YXw+UnHYinPmY0izWixf2+NcR8hfCKcMoPL8KgYrydbIe5uGd+CQYWHFIrcoYyWw5w73lzMp8t38NW6XfxzSssxTNLR1tLrp1Df7Cb5H+R091KKRIAnw5cAML/gZvoIa1q1pFAECEkP9VL7Nj7IVR6nhalMaNZnsDh6jPnYCCEZHgbrZl8mKgAodcU8EsnZQ+GUG2tjsavTYsVO/FtDfBG3vbKYTiJ1r5t5FXvjnhuvqS4HFXHboWWj2YX1sskeKiCY8r0NWjwtvjSGYyw8pGtazDotzoyPxIq+uf6sMyEPbpFpheKgor4CHKZs3HUgrkHhj1/4phlXkx2vWtKZ+7s2sy0ar1/4kftL8/GN7g+5K3wDACPFKrqKGlbLI01Py398j8ed60nYeOulj0IRjMtGMcJDdVK7hrVBntEt+dHQleZ8u/CQsZHlTtOSOXsok3dnh+xED+G8gbrxmgxPS+J7lw2F+nu6Lprc0dgsn+8kPCRCBKIpPC1GKrIImWFCOwyjxSsiILMPDx0gvg5Orr1qCsXhjPK0HKac+diX/GS8fbZGS6e/a4v5+CXfo7QjvhXA85HzzMfDXTHh6ACX5lGZER1qGi1W9spiM/RjYBgC1pCUIcQ1rhGUnqRGfKtlrAZMUHpMAagxxy7VtilkTHkWoYzaimraxHmxMmF4doxNOpt06UQK9Pet3kaPYhoLDorLpQsP9dQNsn5ia9oMLsNIMoW4IjujJVGndLA1LQrFoYwyWhQHnXcXNr7Dcju9e7KVk12x5oxvR0YxKXIG30b7synalb5manMsBXmfLDbDQ1ZqZBFtRMAUhXoI6/U64lOBu+spz0ZYJL48fKyvkEFi4Tavgzot2Ta9y1ynJbOO5gB+ikUWRkuCELcp4SHDEGywMSZzJcQtc5UD8KLvMa72fJHSuEkOD+nZQ1mU4z8/8Bfz8cEODykUhzLKaFEcdN5btLXRNVqO0L8tT48MNcfGet4D4O+hK/lNaCxRXFwZvI+3o6fhFyHTCDHEng342KqnI1sxmim20TUr1swka0rzzz0fARDVf32sQly77sGxGigxr41hQOQq0SOEW8vmSSFo8DmoF3JAFtLFRrycCsNIaZA+IlI0qYx/R11Lk9gsESypxhnrtEj8BGlIYSQ8ELo27nkqb5dxP+PzMjwuThsmQixMCCo8pFDkEmW0KHKKIIqfIMeLNUmpwgYRCSsraxp1faM42ITIWfwppGlVjHootQkCyHpdc2IYH1ajpZLOvBY+M27+djQRriHM9SfoUFIR6x4sLam0sQ2uISG92pcmhNFYTN1HihBKOg2HQQ1FFIhQXGXhdBhGSxCv7ulpfHioWA9L7cPOaHFWXM5DBLeQBGy8aABVdIh7nur9MO7nScgeclqnBWCH7EhYGkbtwTVaVMqz4lBGGS2KnDLf/wtWF1xvpgj/wft6o6/l1+u3WolpH/y8FvkeVbI9xbpnpCEhLJBoLBSIICHpNtNc7wrfQP+G8WyRmpB3j15fY5L/wbjztLWkMVqkB5eQeIiY86xGSYO+iRaIoHmtYB40LQBEgrablpOMpT2ynb4+Z0aLz5JVk0kInPFaupcmZKP/CDssLneU2AnAca5UNYUStCZZhoeyKRIXwsNaqYmKA1l4aHKBUOXlFIcwymhR5ABpbnSdRLzmJCpd9GB3xit0ZR9/8bwQJ6od5/0X3/jHxs0zNn7DtR/ASztxQB+LN1oMUaeRFl1IMM6wkbgI4OP0wD+5KXgbSy1pzdZ7QcxosdZ0MYhYwgk+G6MlVhNGu55V05KtdiUVmXQfToS4ie9XJmJGizujpiYTMa+N3RoFQenOqGk5RlQCsCThc7RyZuBxxofPAmIap0RixeUaV6fFwPD87bMpmKdQKBqHMloUTeZBz3hWF1xPMcmdn09xr2BOwa/ijBFpo7sY63mPqz1fcIZrsTn2PfdCOon93O2ZaI5ZQzygeTmOFFX6WPw3Z8PDYaTTFqTQO0Rx8Wl0JLV6OfrYvSzhIV2rYueFuMj9FQCnu5bENSY016Hfs4Ag159aEuf1yNV34mCGEIoTIa4RTjPer0x4LaLjsKWNQGOwem3sCOPJGB4yjIqvLAXzEtkoe7JZdgPgKLHD/jq64NboOZRtnRaDGr3I4YpoSVbnKRSK1Cij5TDi6/W7Kbnzw0brSQw0YWvM8LjW8zkAH/juSnlOV7Ev7TV/ojcZNMSwVn7m+dh87E8wWo5xbTf75yRmjcTCQ7rBIYI0yOTsFIP3It+Je27cKyKFJaVZ28CeDF9szquQ3QEoFvX009Ox62SsVoexrgIRpEOR1rQvHw0TtcWmMloya1qy9rTohlxIeuJq1TQGbwbDIIQ7o9GSrgmilTWyN5DsFTQIJ1TENSoNO22YaPCb0FguD9zLNzK547RCoWgcymg5jPh0uVYddvxX5U26zgTv31jh/6n5vCKqfXM92hX/zfVnwVgX5ke9z1JecDW3uN9Je+07U2pgNCPJCNk02IgtE8eMsEx3PeOoICE8lMhK2Yf/hM+lVhbG3auWItNoMeq4bLFkHz0W1grJdaSWX3gmm+eY6zAFwQE6F3nwikhGAyJbzHTcSNA2gchZyrP2uosd1mqxGhpB6WlSl+dMnpYgnozhoTYJlYpTsSbaO+3xUIKmpTFCXNAqDs+TA7I6R6FQpEcZLYcRhnzijXmbm3Qdo1y+wTaZ3FH33tB1TImeYD4/Xi/y9lvvm3FrSU38zmtsIEbxs/16jZSRDf9mamQ4X0SGsUyWxJ1jGBov+h4DoA2BlOmwBg14zRDUZe6ZAHQQBxioF6YzysxbPTZGmu5d3tfMMauBYKy1HfW0120mQ3+Tq0yPjJ4WEc6oadmray86pPBAJGLVoeRK05LKMAjjySjENUJ4djV4rKQzXLU1NK1hokKhyB+qVKOiybQX8RVpLw/ca37DvCTwAO/478t4jVpZSFu9sFlX9pqF2wx8hAnjMfv+GF6BnXTkhtDvbK9phAEAznV9w5FiZ9yYHQ3Sh1dEcBMxjRYDD2Hb4nGJqbQfR06Me16tGwNtRR3uqGbs5drTkqk/jxNPi1FbJPHzTIXf4h1pqqbFK8KEpBuZ4ntUSOYuPGQYLZujybV6IDk8ZHbPVn8uFYpmR3laDiNykQpp3ZgMgWJHUcvMSEz8aHXxL5J9zWwNKw98sMLyTJrGCMDcgl9yg/vjuPnGhlQoggSkx7Y7byKbdK0JwA/dX9JbVJn6k1QkpklbKSJgyQ6KvcYoLj6PHG8+3ynjjRjDwPETwh0Nxo3likwpz34nmpaEujaZSPS0pGvY6ORa6YSuIQdCXDsRtB0BfNwc/DWXBh9IfS8sZfwbGR5qNlTGs+IQRhktiqxoa8kQ6iN28KL37/QUe1gs+3Jj8Ha2yC6skkeZcyQu3o2MSrrOtxtjjfn8hHCL+HDQb7yTAJgcOQmIbUiFBDK69+0oFAH8Imx6PVJh6GCs5f//L3wRoG3m/hTf5p8OX2g+LnNVxB0zDBQfIVy6pyWWPZSbHSacoWqsk4J26Qw2O6zekSBN17SkMwpCDjQtmXQxVj6OnpTkITOIZQ/Fd3lW4SGFovlRRsthxLwK5x18U9FPxPoGTfX/ju+6FwGwXXbis+iJjAo8mZTFs0geG/c8UShqeFmeCf8g6X5zo1qYychUOd21hPYiObU6E4a41K4hnxVjY7/aPdUcM+psFIkGM0MmUR9i7ex7Z+jncceMb/5+QriiRpn/HAtx04aHpKPsoQYze8hZyrO1NUBIeppUxt+bYX1h3BkbJho/I00NvYVSZQ8po0WhaHbyZrQIIV4UQuwUQiyzjN0vhNgqhFik/zvPcuyPQoh1QojVQoizLePn6GPrhBB35mu9hwNLtjjvK2NHX7GVN/x/tj22Q3ZMe+59oetYH+2pPUmwWoyMnH02fWeMMcPD0ddVmdWaDdrpHqJMXhojA2kvbc0xwyCxhocSPS3WQmXr9UqosWMuAtKDX4RwRYzwUG6FuKYRZGO0eIjgEjJjFd4IbgLSQ6FwHh4yDISmh4ciDsJDmT0twTS6GKckVsT1EtGzs1pH3KV1rFKhaBz59LSMB86xGX9CSjlM//cRgBCiDPgRMEg/Z5wQwi2EcAP/Bs4FyoCr9LmKZuB+z4SUx7bLTmnPnRA5m7cipwNaRVgrRjGzfcSHboLSbRoZ6croO+FIl1aALlNmiREeutnzAQCfR04w05sLCVh0E/EGwBp5JBALZyUS0PsTJQpxuxSnT891SrrsoUw1UKw04HPsafETinlaciLETb2+oANNi2ZENV0rFE7QtHiIKC+LQtFCyJscXko5QwhR4nD6RcDrUsoAsFEIsQ4YqR9bJ6XcACCEeF2fu8L+Mop8spt2SWMfRUbyQvg8lsrUpdMNDAPEK+M3RcPTsle2jRv3iYhFD6JtIEujJezM4NWxIyoFLiFpyFDDI3FzejR8panjKRJWT0vyr06/hpcJp/geEMSra1qC5nOAzsXZ63PsMA0Sm7RgpwJVgDoKzM8jE4melqZWxE1nVIUdZg9lW7XWjlhX7lj2UKsR4ZJYLEChOLRw7GkRQowSQvxEf9xVCHF0I+95ixBiiR4+MnafXoC1eMgWfSzVuN36bhRCzBNCzKuqqmrk0hTp6ClimpigLlaslUUskKWOzjcNEBnv5jdqvtRQxBWBe+OOBS0iVu3/zNoMO1y60DdTeMidICYN4DW9L3/1vmDWArH7Rh/CkzI0YXhaDKPF6J2U+zL+ySEUp6nAAPtlIW2Ew+JyltovoSYKcZ1lDzkID+XAaImFh5SnRaFoaTgyWoQQ9wF/AP6oD3mBVxpxv6eBvsAwoBJ43LiFzVyZZjx5UMrnpJQjpJQjuna1r7+gaBpGYberg3/Cp4sTMwlbrRh6EW+i0WLUXpEFzJUD2Co7A1rWjrEpGgJYJ/VG0pEpPORJaIjYIH3U6NVte4td/Mz9EQCBLIW0AenFL0K4I4anJf352Roz6cJDfuHc07KfQlP/k4k4IW4OyvinW18Id8biclpTyKaHhyL6n8VY9lBE1WhRKFoITn8TLwGGAwsApJTbhBBt05+SjJTSrPMuhHgemKw/3QIcaZnaGzByTlONKxxQcueHXDXySP5yyZCsz73ENROB5O2opkXxEOajyEhmWxrSZap7YsXwtGhGS2xz6ab3JTLKr38n8BRdqGY3bRkiNgIWT4sIE4w2fgPJVOLdKLBm0IAvro9Qf723ULaGU0zTEh8eyhUhixA3MY3arC3jYEOvlYUUO/W0xGUPNVHT4sjTkt6YypWnxegqbRaXE5FWFR5SQlzFoYzT8FBQaq15JYAQIjnNwwFCiJ6Wp5cARmbR+8CPhBB+PezUcrSv0gAAIABJREFUD/gWmAv0E0IcLYTwoYl132/MvQ9n/vvtZv77bfal+5/wPc0/fM+Yz/2WWh/nBB7hjuAveClip7W2J5Wm5RHvC0CsuBnALtrr9T/iNS3Z6hZODzzBfovRsTZDRdw50YH8OjiWDdEegKbxsNsIs90cEzUt+Swul0gq8bAdBygwe/hkwmcRvjZZ05JBiBt21DAxV0aLJsY1wl0e0q9NoVAcPJz+Jv5PCPEs0EEI8XPgp8Dz6U4QQvwXGA10EUJsAe4DRgshhqEZP+XATQBSyuVCiP+hCWzDwC+llBH9OrcAnwJu4EUp5fKsXqECIOvOzn3E9qQxvwgRiGqb1Cp5VFwROSek0rQY2HlBYqJIvVMz4aw2/E2yO4MDLzJIbKSzqEmqIZOM4L3oKOYF+9NeHCCER+9qnbiuxnlaEoW4uSKYprhcOvFw8nW85vxMWAvC5ULTsl9vzWCHs4q4TQsdWrG2JVCaFoWi5eDIaJFSPiaEGAPUAP2Be6WUn2c45yqb4f+kmf8w8LDN+EfAR07WqYjxx7eX0MYX+3hf/aYizexkOlGbNJapS3ImjPBEoqbFoJrkarXGJmQIYDNpH1KxXB6dVVrFVrqyVU91thPXOmkjYCUgvRSKYJIQN1eELZ4WmfBCsxHiBqUHn8uZx8QrwjTImKelqWX802paZGZNS6YQUzaEcMfXaVFGi0LRIsj4G67XSvlUSvl9IK2homg5JIaDolnmQbYRySECa3ioMZjhoah9Sm3UxjgwBK9GnZZcfpvOhuXRPgxyZWf4WQngpQP7ERGjjP/Bq4ibqoqvHdl4WjyECenekWBT67Tkoricg07WTrGGu1qbpyVXBQsVipZIRk2LHqapE0K0PwjrUTSRu99dypwNu5t8nVs87yaMSPyEmuRpMf7wHx1aFze+OdqVt2z6EwFxdVrcRHALmfMS+E5wNbH6hWYMhJOEuCJHO0zIUXjIgacFj9m9ORM+i6ERlh6tf1S0cSGi3DRMzHF4SMTqtKjsIYWiZeD0N7EBWCqE+Bww+9ZLKW/Ny6oUjeaVOZt4Zc6mJl/nZNdKyzOtd41LyCaFNXrodV4ur53IbznXHPeJEMGo/XWNTaijqM1q8801LhtdSzYYmpZAnjQtRsaLz8Yb4cd5ynMQj/k+3+55k0tcs4DzbecmFpcDNG+IK7U2JRWZisuFHAlx03trsiEkY+EhjwoPKRQtBqfZQx8C9wAzgPmWf4pmZOmWar5pglelJ6nPfdvi+fATMsMzTQkPzY4OAmBaUXzGUbqsD2Nz/5Xn3awqu+Yad1ONFr1OiysSJCTdZihMJnaPbAKpvBG+LD47wyMEkl973uEoV+pCjVbvSDohsBO8GUI7zorLhXIWdtOyh/TwkIiYnZ8VCkXz4lSIO0FPOTZKn66WUjatGYyiyfzg/2YBUP6I/TfhdFzj/ow/e8cDUNLwWtq5BQRNgyFz9k1qGvARlYI9rvg+Renc+ladS3N6WuZEB3Ksq/ElgjRPS5C6aDDOG+Bx5679l2G0JNZp8WZh7A0Qm3EJSQEWAyEaAVfypu0VYcLRWPYQ0HijxUmXZxmBaBRc9u9Z7rOHYkLcpoRF80HbAg+1DfZhvMTPX6E4lHBaEXc0sBateeE4YI0Q4vQ8rkuRZ052xdo3nepalnS8fSwKSCEBM3unafVFhBZ+SLB3ndTX0LokN5+n5YHwdXw38Fijz49pWgJNMvzSkcobYQhxndQaGePWHKilYkts8IC9tyWxIi6Q0RuSikyaFtMLkyaDyCsiORTiWsNDuctKUigUTcPp17zHgbOklGdIKU8HzgaeyN+yFPlmnYy1cHrN95ek4+1EzGgpEEH8+jfvhgxl8DMRwIvHkp0iiOqbTXpjyC/CMU9LMwhxQ3jYII9o9PlmnZZIMG9GV6pOyNl4qP4UugFIaBxZY+9h8hAxX4s5v5FGS8aGiQ7CT7lqmGjcz6ppaXHZQ6orouIwxanR4pVSrjaeSCnXQDP46BU549eed8zHb0VOSzrengNEpeZmLiJAQQ40Ldr5vrg6LU6qta6MakXsvA7mtlQC0otXRHBH6gnlaf0hmcLTkkV4aIPUila3FwcIS/3PQ3C/7dw4TYvpCWlc2nPm3kOZPTnWCr1NxdpLSdVpUShaDk6NlnlCiP8IIUbr/55HCXFbLb3Fzrjnl7lnkvjVrafYQxValns3sdf0tDQ1tBHEg9cSHvKbXoDUG9b06FAC0tusQtymYmymntD+nJfwN0gZHsrC01IttQ4d7TgQ8y4ED9jO9eYoPGSksqcLX5nXT2MUaZqW/GQPtThPi0JxmOLUaLkZWA7cCvwardz+L/K1KEV+OU5sSBq7w/Om+dhDmLainsXRvvrzqKlpaXJ4SHptPS3pNvJaWYRfhMxGfs3pabkqeBeXBu7P+jyjG7YvuC9v60/VnycbLVCt3tG6naiLbdQBO0+LxC9yY7QYHrSMdVrSXl9bT257D1myh1qaoZxGa6uKyykOZZz+JnqAf0kp/wFmldz07XIVB51V2531F/q370kARjb8m28LfgnAcLHWPN4OrZtupdSyfIoIENbt26Z6CYJ48WI1WjJ7WoyeNEZrgUz6l3zytZ62nS27pea1KmjYSZCsG6Q7IpjG06JVFs68mxkViH2EiZA6PGR4IULSyB5qfMqzz4nRItNrZoxMn1z9bITisofCsfu3AnKYRa9QtDicelqmQlw3s0JgSu6Xo2gKNfXZ6Ql20pFTGp4CYJR7Od91LQA0PQNApewMwJO+/8uhpsUbFx7yCUNcm/q6+6VutAjNKGuN4aEa3YNRWL89b56W1OEh51qPWFdti6jVxmhJ9I7kwtOSPuXZuL79z3g2ad1O6NS2jQoPKRQtEKdGS4GU0vzLpT8uys+SFPlmdbQ3n0VOAGA37czxp73/BOD/uTV71LrRHSu2AjS5XoVmtNgJcTN7WjqbRkvrE+LWyZhj0sn6G+PiD0kPRMNJ52aj9TDmeQkjDJ2TjabF8EKYRouhR2mEp8VJeCiYwSgyPHa5yh6SrljXam8rM1pUeEhxKOPUaDkghDjeeCKEGAHU52dJivwiOVJUsVl2A+I3UEP7cIJrDQDrLSm+v/VqmpemdifWNC0WT4sDkajhpbjN8zYAbVrhj16F7GE+Ppamt1mwQyt1H0wKDzTO0xKmDXrTzEByx+9Ez0ZTist5zToyqQ0D02hIUafFieGTDVER3zBR1WlRKFoGTn8TbwPeFEJsQ0szOQL4Yd5WpcgbnailSATYIrvYHi+mjmGu9YCWtZNITjwtumYGnHlalkdL4p6vlH2atIbmYBfteStyGpe5Z7KF7nm5R0pNiwg5rm0TwUVUCvwiRIEeurP3tMQbCZk8IelwpGnJYBTluvBgWHjMhokewq3K06JQHMqk9bQIIU4UQvSQUs4FBgBvAGHgE2DjQVifIsf0Flp1U8PTAvBcONYG4DeeSZbZyX7mpgtxPXisKc/CuacFoEF6zQyX1sYdoZuZct5MbuHOvFw/Xe8h5yE1rWpxsdWbZadpSfCONKW4nCmizdQwEVIaLabnJ0eFByPCyB6S+ISq06JQtBQyhYeeBTPV4xTgT2il/PcCz+VxXYo8caRutGyRXc2xv4R/zGOhKwD4qeeTuPnHNLzC57r+BXJVXM4aHnKy2Qj2ymIAPo2e2KT7NzeBgq7sz5PRFcID4UDSuJM2CVaCeGkrrEZLsqfF+NzCieGhRhSXc5TyLNN7cnJdeDAiPHiJmI0yVcNEhaJlkMlocUsp9+iPfwg8J6V8S0p5D3BsfpemyAe9TaMlPjyUKuwTxcUKSzimqfUqAtKLTyanPGcyhh4Na9FIq6BVEU8It63mI9tKsQE8tLWE8Ow1LfHekaaEh7Kq05LCKMq1EDeil/E3xLgtrk6LSmtWHKZkNFqEEMZv6/eALyzHWthvscIJR4qd7JXFSd/2v46WpTxnQ7RHymPZkhgeclLGH2B9VBMFL5NH52wthxpaGf/kTT1bT0so0Wix8bR4TEPDHbs3NE7TIpwYLenDT7mulmx4WrwJr7M1oLKHFIcymX7D/wtMF0LsQssWmgkghDgWqM7z2hR54Cix01aEuzzBGLg5+GvzcTVtcnb/AD58WRaXA/hGDuQ7Df9iK13TzmvpyDx+RQ7r2UOJ+EQoKw9VUHo51R3rAk4guWhhonjWSUPDVDjRo2SqA+PE8MmGiF4RN+ZpaT1Gi0JxKJPW0yKlfBi4AxgPjJLSTKZ0Ab/K79IU2SIdlMI8SuyME+Gm4uPoSblYUhJJnhYj68NBKnVrN1hSUdazXeZJDgjhgWgoRZ0W5+GhJANyz8YkYySpTkuew0Mxoyh9cbmmpuQbRIUHt5Bmb6zWZLQIB5WPFYrWSsY6LVLKOVLKd6SUByxja6SUC/K7NEXukfQUe9iUwmh5KXy27Xgu/wQGpBcPEdz6pufU06LITOrsoeyFuAY7ZAeQEajZGjcnMXsopmk5NIrLhfWoeKEI5PS6CoWiaTgtLqc4BOhILX4RYofsaHv8gfB1VMpOjAtfGDc+LTqMe0PXMbDhxSavwRDcGptMNh2IDxXyFSAKpQoPZelpsW7Qn0b0bK2Epompy/g3pfdQmuJyMlNxucxp09kQMYwWPZSpsocUipaB+vpwGNFVaDKkKtkh5ZxTAv9nMyp4OWLvhckWa8XVesCfYwHl4YwWHgondczziTDBaDaeltjcr6KDuZbPk1KpEzUtRlE6V556D2UyinKePSQ0I6UQw9OijBaFoiWgPC2HECJD2kB3sRfQqrM2F4anxdAKmA0TDyOjJfFTypXnxcjgcROv+8jW02LVhdQafVJDdXFzPEmeDZHS05MJo/JsyJEQN4XRkuOKuIanpUDonhZltCgULQJltBxCZBLiXuz+CoCKaH7KyDvBENwaxoqPMEHpRqofxSZjeAPcMt5o8WetaYnNrZF6anzlorg5djqUVJqaTDir0+Iw5TlHQtwIRnhI87Qoo0WhaBmoneIw4jL3TAAq6dRsa0jytGTpBVCkxiiA5pHJnpZsKhkbn0dQulOGEmNC3ASjJYXmJB1Z9R7K2DAxN8ZFLDwUzLg2hUJx8Mib0SKEeFEIsVMIscwy1kkI8bkQYq3+f0d9XAghnhRCrBNCLEnoKH2dPn+tEOK6fK338KL5UiKNzbMAi6dFbQg5wXgf4z0tslHF5QB8IkIdBbZz7MSzoRQNGzPhRNOSqQ5Mrsr4n3JMZ+02hqdFtD5PiyoupziUyaenZTxwTsLYncBUKWU/YKr+HOBcoJ/+70bgadCMHOA+4CRgJHCfYegosmefbMOE8JhmXUPQJntIeVpyg7GxWjUtHiK4hMwqbFJgKf5nemgShLj24SF33sJDIMDlPfgpz6iUZ4WiJZE3o0VKOQPYkzB8ETBBfzwBuNgy/rLUmAN0EEL0BM4GPpdS7pFS7gU+J9kQUuikV7T8//bOO8yN8trD71Hb9a577w0XsE1xwXQwzdiU4BC4oYSSBLhcSIOQUHNDQopvQhpJICHthhsI5CYQCBDAdLhUQxybAMbGGGww2MbdW9S++8fMqI6k0e5oJe2e93n0SPr0aebT7Kzm6JzfOcfQhxZ2+FjdtiOkwkOOpkXivnXmrQe8Fv4qJap2IyXEzfC0ePFi5HJccCkAbyVHpN/nxWgxHfS0SJykERKlvo6C4YoLcZ3DnsxJeY5pyrOi1ARdrWkZZozZAGDfO1XORgHrMuatt8cKjechIheKyFIRWbpp0ybfF17vNNFOUAw7Ta+qrsMxUNTT4j9OqKYjvZ3cWGVGAwLBCCRyjZbsiripxx2s02Jtp4ShVsxosQXdnQ19Olr2hNRveEhRujO1IsR1+6YxRcbzB425xRgzxxgzZ8iQ7lnuvTP0phUgr1FiV9Nud5N2hLgNZXYgVgoTc9G0dKZ4X+pCHWyAeLYHxU342pnsIU8ekkC4qBDXz/MoYX8uJ1RWc12eFaWH0tVGy4d22Af7fqM9vh4YkzFvNPB+kXHFhVffK9zDso9YdTZ2VdnT0pZRXM66j6kQ1yfcjJYGKT+r5rLoRQAISWsg1JDvaZE4sZxU9WhH67QQ97a+YKSopsUP3YkTHkrkaVrU06IotUBXGy33Ak4G0LnAPRnj59hZRAcC2+3w0UPAfBEZYAtw59tjigvfuv/1gq/1sT0tO6hyeMi+sDTYv2AjxMtKx1UK43hGQuR7WsppJOhkDAUcp2bI3dOSGzKJdzB7KELcmycjGC7aMNFPsWxeGX81WhSlJqhkyvMfgeeAqSKyXkQ+CywGjhWRVcCx9nOAB4A1wGrgV8DFAMaYLcD1wEv27Zv2mFImvcUOD1XZ09Ju7PCQpCvi9iQhbiVxS3nuiKYl3crdfhSMQLwta07ExUiwhLjlh4caJUqrfV4UJVg4e6hB/E2dd4yWJnH3tDxy2eG+7UtRFO9U7GphjDmjwEtHu8w1wCUFtvNboPOd+no4g7FCR1vpU9V1OBeWzPDQNnpXc0ldinGRZJWqZOwVp6mfu6bF+7+6saVkaU9Lo4sQN99I6GgZ/0aitOHFaCkcHnIKJ/qF4/lpdCri5hjWk4ZW9/9IUXoqtSLEVUrw7kct/OLJtzr8/k8EnwJggxnk15I6RLoibjo8pJoWf3A8H27hoXI8LU7qcVrTEskLD4VI5Hlaoh2siNtIlFYvRkvAbgjZBWh4SFFqE71a1Aln//YF3vmohU/MGs2QPg1lv/+woFWYuFCF064i6irEVU2LH6R7D2WkPDv1S8oIwW01lhdhtbGrCwQLCHHzPC0hSOwue92NxFJZZUUp4ml5OTmZ3ca/c9vJHupVIDxUy3Skxo+i1AvqaalhYolkKnSws826+ATK/D4aLxtY23im30vrMM7FYIDsAnpecblKks4eSqTGOuJpecVM4fzol7kxfoo1EGrIKy7nqmnpoBC3UaK0eda0FKrT4k8WWn72kHpaFKWWUKOlRtnVHmfyNX/nxkdXA7Blt/XluXx9flrz+9taaYsl8sYBrgjdUblFdoKZgVWAelr8xDEiTEYop6EDFXEBHknOTnvlXIyWMImUhiZr//HyjZYGYh41LYWNloYym0IWIlVcLkfTUmtl/P1RQSlK/aFGS42yrcX68v/T0nVZ48+v+Shv7sGLH+OCW5e6bmde4J+px7+JL/RxhZ1jvbEKADaopoX+Tf4YbU4II+AixO3UBT3YkOdBcUsxjnawjH8DUW/rC0YKamYiPheXSwacirjqaVGUWkKNljqjULz66VWbXcf/kDgm9Xh5ckJF1lQuHwSGpy4CEZ9+IdczPz1jVulJHnB6D2WW8e9I76E8QhHX3kN+hYcaJOatjkyxhokSK6spZCkSYp2fTban5a3vnuTbthVF6Tg9+yduHWIwfLijjUTSm4O4mVY2mb58Kno1K83YCq/OGzEJ2x4A0+Oyh9waJnZEWO2Gk6ab2eU5LcTtpKfFJTyUK06NdrCMf4PnlOfCxeX8Po/iYh2vRtqJmiARFbcqSk3Qc64Wdcp721o56LuPpp7/8sk1/PLJNZ7f31va2GmaasZgAYgRpoE4IRIExPj6C7kUE4c0s2ZT+Rku9UA01TCxc3Va8ihQxj/X0OiwENert61IcTm/tVGOpiUiCVpMgxeTSlGULkDDQzXKS2vThX83bG8rMrM4vWllV5VL9+cSlxANRFNpz10ZHrrhtH27bF9dTcrT0smKuHkEw7A7O/zoqmlxjJYyi+WVpWmpsBDXIVPDonoWRakd1GipQd7evJtL7/xn6YkeaJY2dle5dH8uUYkQkXinOhB3lO7s5I+5hYfsY9yp7JfobjAJMnNW3PoFWZoaA0n3TDY3AiSJSMJbynOgUPE6Q4PE6UOL5/0WwokCJQmSNNaTeqrRoijdHTVaapAdreXrAgpheVqqW1AulxiWpiXih0i0Drnj3w+syHZjLuEhp8tzp45x82AAgk6FXOwy/m4pz1BWiMipjOzd05K/7SF2i4rhstXzfr3gHE/1tChK7aBGSzemgSjTA++wuyaNljgRu2liV2paaoHpI/tVaMsCgVBe76F2E6JTPqYmy2gJZ3hwQiTyPS0dMFrKCl8VEOI6zTcfTs72vF8vOMaKHzVazj/U38y97uwxVJRiqNHSjVnZeB4Ai4LPVnchOTjZQw1+iESVbIKRnN5DPtQvCUZS23IIu3RVTj0vI4OofKMl3yBK1aLx2fh1jJbcInqKolQPNVpqED+yK4N41xV0NZbREvdHJKpkEwjneVo6bRQGrb9POMcYcq3TAmV6WhzNjQfDoEBxuUqcRyJpI0zDQ4pSO6jR0k0ZRjq+f2N8URVXkk+cMBGJZVRr7TmeFlPpAuzBXKPFP09LptESJp53MY85Homc9OhihMtp6BgIW12ec7KTHF2Mnx47Y9LZWLUoxNUy/kpPpedcLXoYEwIbADgv+hWeSM6s8mqyqaanpVa+7O//wqH061WBzx0MZ9dpkVjnG1I6RovEUwewYEVcKCs8FLY9gp50I7bHh0TMqtJr42fqfGbxv7gJgqinRVFqCfW0dFNui3wXgG2mT5VXkk8qeyglxO15tvP0kf0YPaCprPdcffyepSfleVp8KLpmGwuRHE/L0AF9s6ZFOxAectKUW/BQFThltGRvv1KC7nT2UM87PxWlVlGjpZuzld7VXkIeMQnRoJqWyhAIZ/UeipDwQdPihIccnZQhIgkOnTIia1pHNC29pRWAHabZ8zryjJYKpc6ns4fU06IotYIaLd2U5xLTAHjHDK/ySvKJEaZBMrOH1GjxjWCEgEmLsP3xtGRrWkK28SLh7IJwqf2UFR4qw+CwOy+TzE57rlSRQsfD4lf20C1n+5uSrSg9ETVaahC3pnrl0iytPJbYz4fV+E+r7cZvxmpPoCnPPhIM5XhafGgkmJM9lBLkBnJ6D5nyPS3laVrcPS1TZL017PPXmd/F5eZP75ofEFrDRenOqNFSg/iRYdKfXWyrwdAQYBc7S4cGepKnpTMGqaeWPsFITpfnWOfrlzh1WiTHaAnmCnGd7KHyU5691WlxjJZsT852rNDSVuPv+R5LZQ+pUa0otYIaLd2UgbKT7V50AlXAuUD1xjJa2rtQiFtmL7/6IxjJ6/Lsn6Ylbm/T8o6YQG54qAPF5WxDqOzsocxhu72A38bvnMCbgGYPKUotoUZLDdLZ8FCQBL2lja01mDkElqYFoE8P9LRUnLwy/vmpyWVTIDxkgtl/t06V8fdiuDr7yykwV+keVn4Icf0oGKkoihot3ZImrOJeu72kkVYB5+LS2053VU2Lj+SV8a+AEFec8FAho6VjQtzZ4wbQt7HIuRBwT3kOVygL7YWklWKu4SFFqR3UaOmGNNkC15Yaa5To4HhaeqKmpeLk1mmRuG/F5SKp8JDtaSkYHipHiJsOD33h6MncfkGRDtgpTUtO9pDESBrxXYj7RnIMAIPY0eltdfuwpKJ0EWq01CCddSU3i2W07Da1abS0pzwtrSSMkOhCzUC3d9PnVsT1sbhcOjxka1pyPS2OcRT3Xsa/PCGuu1GUzpDy94/bjmUkHR5c4ds29xzuT8jWqBWk9FDUaKlBOvt9VOuelmiGpqWrvSzd/rs+EM7KHmrwJeXZKeNvGSvplOfOh4fSQtxgaZOjQMpzgx+GmQu3JY72fZtThtWmzkxR6oWqGC0islZEVojIMhFZao8NFJElIrLKvh9gj4uI3Cgiq0VkuYjMqsaa64nmlKalNo2WWEb2kOpZvDNhsLeqsbmelk735MnRtKSEuAWNlvKEuFETxJOXxNlfTnG5MHHfm24aDB+YgQB8Pvo5H7dbebq9N1Hp0VTT03KkMWY/Y8wc+/mVwKPGmMnAo/ZzgIXAZPt2IXBzl6+0i+nsl06THR5qMbUpxG3PMlpUz+KV+dOHc9/nDy0+KRgmEXeMBuNrcbmUpkX81bR4PgcK9R7yI0PKxvnfM8YKD41vu52/JQ/2ZdtdwdkHjmPxKftUexmKUjFqKTx0MvB7+/HvgUUZ47cai+eB/iIywm0DioVTabZWPS1RSQtx/ejMW090tnDgjFH9ik8IhlPZQyESBMR0vpFggTL+hPzJHvJscBQoLmd1su5Z51Ehrl80g+H9avP/XlH8oFpGiwEeFpGXReRCe2yYMWYDgH0/1B4fBazLeO96eywLEblQRJaKyNJNmzZVcOm1Ty+xwkMtNSrEdbJZ+tDSbTs8r118QnV2HIykPCJl9fUpsc3M7YULZA8lCYAEy66I691ocS8u54s3qc4QjQEpPZRq/acfYox5X0SGAktE5I0ic93+O/N+rhpjbgFuAZgzZ053l1sWpeY9Lfav4ogk9Bey3wRCKU+Ib120A5bmJCzZKc+5dVqssUh5RouU0YW6YHE5/4W4ahMoSm1SFU+LMeZ9+34jcDcwF/jQCfvY9xvt6euBMRlvHw2833WrrT+c7KHWGi8ul/u4a+jm9mwwklFu36fuxyKuHhwTcPnbBSPlZQ8R8+5tK1Bcrid6WhSlp9LlRouINItIH+cxMB94FbgXONeedi5wj/34XuAcO4voQGC7E0ZS3Bkgu2gxDTUrcs1s4LdP4O0qrqQbEgwTkQRgaBAfy9tnGEPpMv4Rl3nhCgpxC2la/BPiVgrHc3POQeOquxBFqXOq8Z8+DLjbjsmGgNuNMQ+KyEvAn0Tks8C7wGn2/AeA44HVQAvw6a5fcn0xWLaz2fSt9jIK4nd6aj3R2b5SJbFDKCESaU+LHyG4YDhttIh7nRZrXnnhIUuIaxUXLOkDC7oLfSPE2GV6ed6nFzLr+fhVEA5g//EDmTy0N6s27vJtm30aQ+xsi5eeqCjdgC6/ehhj1gD7uox/BORVczJW6cdLumBp3Yapsp42XH4F1wiZ4YC7EiVSeJWyePHdnczFMVoq5WlxKuIW8rSUEx5Ke0mKVXnt3xQuWFyu0uFVReTEAAAdh0lEQVShQb1r938JYHDvBjValB5DLaU8Kz4xLfAOUwLvVXsZBWknnAoRtZnaviDUGw++vgWwLuRpTYs/RkskpyJubhl/Z155Qtx0eKiYp2V438a00ZIjxC0rxNRN0DL+Sk9FjZZuhlOjY4vpXeWVFCaJsAmr3kitioXrlRb7ePam1T8hLmSHh1Jl/N08LWUaLWUJcQuHh1SIqyg9AzVauhnj5EMA1pmhJWZWD2NgtGwGoE17D/nKDtMEWIX7nMq1vtTCcRHiuqc8ly/E9SyiFbF0NG5C3ArW+/Fbh7TniNrVmylKraNGSzdjhFjhge/EzqrySgqTzLAcWmu01UC9cMrM7DqLjgEQzgoP+etpcYwhP+q0hEnXaQl4KY7iYhRVok5LJfneJ7TMvqJ0FDVaapDOFLYaaXsw3mOQT6vxn6SB/4kfA0BrDQuG6xHn4t1AjAafhbiZdVpiJgji8vXRgTotjqHVGEpvb68RffnKcVNdth/Oa5hYb3VaekWC/OC0vFyEDqN18JSehBot3YzhbAVgoxng+7ZP3m+kL9vJFBHWen2NeiOa8rSkU5596e+UER4KkSj8dwt1XIibiQD7jHbpsxRw87TEi37GAU1hll57jOc1KYpSu6jR0s0YKZv5yPSpiLs8EvTndDFZj+v3d+I9lxxS1f0Pao7kZdzEjFXzJCIx/3oPgRUeyijj79RWyZ9XrhA3TtQU2JbH7ZcS4o4e0MTg3h0PQ3a2yWWlqe3VKYq/qNHSzRglm3nPDK72MopjQLrBV+2YgU1V3f9FR+yRN+YYq2HiGUJcfz0tRcWzZdZpKUuIC1aBuUQ6PBQkQVBMUSFuLfYRqsU1KUo9oEZLHTJeNjBL3nR9bbhsrZjR4tcXrQFODD4PwHRZ689Gq0CgwPG4YsGeXbJ/t7+HuxDXL02LU6elSHioAxVxy/IK5my/F1ZHc02dV5SegRotdcj/hBdzV8N1qZosmQyTLXxgBlZhVd4xxnBq9OsALI6fUeXVdJxCqbB7jihc9r3SoQbHQGkg7l+XZ8gr4x8rFNLpQJ0WxwAST9lDkazicn1pAWAHlfN6Vbz1gqIonlGjpQYp9iUZJs6YwCYABrM967Um2ugrrXxYARGunxhgtRnN+Lbb2Ub5fV387AXTKWrwWhatQMrzWb9+nmQgI+XZt/CQISJFvDZuBEJZ2x9up/hvNi6i3U5Qjmk5ZqC/fY8URSmMGi01yEP/+sB1fG9Zw6rGc1LPh0i20eJ8gVfK0+LXL85kN6nw1hwpQ0BqU+lf7Y5+JSKxVJfngqJZj/zf6o94bNW2lBA3lFFbJY8yPC1OD6P2cgrDZaRUxxLJ1P+AX+e84+3xcoo6jqFJQ7q++nQkpF/dSs9Ez/wa5CePrnId3z+wMuv5+aEHuC9yNXvKuwxlKz8J/wyAbdRuCX/oPlVpQz5lU/lJrqbFMgg6byhtbElm12kpZrTEvRktjicoc1tBWyjUGC5wbDOKy+1ojdGA9bicBqEXHDah4GuFdEq1xv9edBAfzygsuPiUvbnv89p8VOn+1N63rlKQhcEXAJjX/gMATg4+y4zAWh5suJIXGy9h78BaAFYmx1Rk/34KcatFJfd9yCR/C/p942PTmVZmyXfHAIgQp5Eo7UUu5sUu3m7bzcweihdMefZexj/sornZc3gfvnTMZH5+1qzC27eLy727pYVGsY2WMhpvTh/pHko6+8BxXHvCNKD2s3smDe3D54+alHp++tyxzBjlb4hMUWoRNVrqhLODD7N/wMoYWmtG8EKycIbKhhquhgt028ISnsrQl8G5B4/ngS8eVtZ7ohlGS3/ZxdYijTN7RbyHZTKNloiU8LQkop7caY7nJnNbIsKXjpnCiH69aAi5GEYZxeW+dOcyGot4WgqVy393S4vr+PWLZjCoubTx84WjJ5ecUynuuvhgHr98XtX2ryjVRo2WOsHpKbQkYf0CvTl+kuu8vyTKu8hVg1xNy/WLZlRpJfVPv17ZIttMIe5AdrK1A0JnN6ImmDJaGojRXqj2SzACGEgmSm4zUkJzs//4AXztxGkcOHEgb3ywk9Ubd2VpZhJJU9RoOWgPy3ifNTZbmF7IaClEri162bFTgLRd9u8u9XJK4WqQeWDW2AFMGNxs7b9DW1CU+kaNljqhiTa2myYuiF0OwFsmXVL/u7EzuDG+CEh3+a0EAz38CvVCrtFy9oHjynq/p9TYKtBV6xrZrzH1+KsLpnLtCXulnhsCxEyQiMQYIDvZYnwyWggRkQRg6MtudtDsPtFpoughROQYQY4BlFu2X0T47KETeH6NZbAf88Mn7fCT9b5NO9tptHUxbkbLmIFNPHzp4VyTcXzA/6SvAyYMZO3iE1xfy/xbZbJgxvCs59fmrLEcavO/QVEqgxotdcJ5oYfpJ+lfiOvMML4X+zeOb/8Ov0ycxHY7DDBe3DOP/GDSUO8C32LZDbX2C/H8Q71rO4rRVRePBTNGpB43RUKcf9jErNetUE6CgbKTrT6JsqMm3dOon+xmuylktNhF3jwYLZlC3LWLT6Ax7MH7kKGZ6dMYolGiRE2QZIGvsinD+hDOEUx7sS39EosXEmsHcxS/R+45lBvPmOnPThWlG6NGS5W45u4VXHzby57mNtHmOn5TYhGvmfEAPJycDcCdiSM9bbPSKZM3nZkWUub+iuzsBaEzxoHfmUsHTfRHPzR/2jBftgO2V4QY/dnFVp88LWmBb4x+7GZ7SU9L6VotbpqWUny4O5kqLrd5V5RGomVlDuWy5jvHZz3321kWso0TL144013S6hSlgqjRUiVue+FdHljxAU+9uYm7/7GeV97d6jrvstCfeK3xMwCcGb264PbWmWGMb7udB5NzPe1/jwrXlujdmL4QDenjb4l1EbjkyPJ1BLXMLefM8Tz3iKlDAJgz3r02SYwQZwYfpbe0scH4Y1Q5hsVA2UFYErSYAn/ToG1AePC09JZWAHbhvTjb46u2ZhlEpTKk3MislROoYI7zxCHNHDvdP2NUURQ1WrqM7a0xLr1zGZt2thNPJFPj5/z2RS6985+cctOzxBNJVm/cmXptT3mXL4T+mnr+bHK6b+uJBMv/sq6VH4IiEKwRXUuum78rOGLKEFZ9eyH7jenv+vpm08/Wn8CfEvN82adjtAzDMq5baXD3eJVjtGAZLTuNd6MlhlURd8V6q6hcL2kvK90Zui6d+bEvz2Ngkz86MEVRLNRo6SJ+9dQa7v7He+z/7UeYdM3fXee8s6WFY374VOr5Z4MPAPDl6EXMbfs5fqgmVn17IecfOoFrT5xW9nsdm6WhQGjpprNmMXFIgbCBjwhSM2LcebbX42P7jiwx019ydRrZWH+pvyYOZidNLMwRfTqUY/g5WUmzA1bhw3VmiPvEMsJDzXbYsxxPS5wgJhHlpJ89AxhmymrWmBEl35fJ+q2tZc2vNGcfOI4Jg7L/bwoJexWlp6NGSxfhpXT90T94MvV4sqzntJBlwPwleRgb8aefUDgY4NoTp2VlAv39i4fxu0/vzw2n7Vv0vU7MvdC1bsyAJhafsg9D+jQwfWR5RdHKxcv19ogpBS6sZXDC3sUviE5tFr8yq/xgr8A6AJ6zPXM/dRF4nnfweC44vIzicrYQ96rwHwH4V3K8+8SQHTaKlzYMeondoblQqMmFKCFa26z37S8rGR/4kL8lD/L8fvB27nSlTXz9ohkVDVMpSnfCh571ihdaY8XqVhgODLxOX3azIjmR3TTyt8g1AFwQvYxK5KU4AsGxA5vYa0Rf9hrRl4073AW/DoNtbUqx/jlzJwzkpWuOAeCqhXtyyKTBvLVpl0+rthDxp5Dbo18+glff254KNbjvrPg2RtgprWMGNrGr3RKWNnnJgnFh39H9+GextZTJUwmruJpbBst1Hysv1Cg5OV8fFjKiG+205bbSn8OpsdJeRkPHOOl6MeeEHqY10Mz9iQM8vx+sirhPr9qcev6bc+fQpzF7DW6/Ma5auCeHTe68Ieyw/Lr5mGT++LC+/mrAFKU7oUZLF/HcWx9lPe9NCyPlI2YH3uSS0D2Mls1577kgehlLkt4FmoVYcd189r7u4awxR4uRSKa/nQv5gp654kieXrWZ/W3hp1d7wSm65bvRgrceMQftMYgn39yUNTa4d9ojsseQ3uwxpHdxo6UEC2YM59bPzOXQSYNpjSWYO2EgcycUbt5XLG38ns8dyvgr7+/wWhwWtX+T/wjdywZKNxE8Zq9hPPL6hyXnvWHGph5/KnoV8UJfHY22zsaD0TK+XwBayusbFCNEWBJ8KfRnTgo+DwdfRusj7rVQCpEb3jx6r7RYtphB3pEicsXo2+hurM2dUOMVrRWlimh4qIvY2pIWJh4aWMHLDf/Bww1X8N3wbwiQ5Nb4sZwRvYYbYqdxd+IQrohd4IvBctfFB+f9igQIBaw/fTyZ/qk3tE9DVonyN7+1kLWLT2D0gCbOmDuWiP2L/cR9rJBJ38bq2bxeNC2n75/fg2miz1lTIsLhU4YQCAjNDSH+bc6Y1NqOnDokL0Q1dXjxFORF++VrY5ojQRZMd9eluLHMTOLfY4U9dH/4bNozccvZs1n97YUlt/mWGcndiUM4I3oNzyT3Ljyxl220tG4ruc2htpSlHE/L8qRVk+ZLobtYkpgNRxbOqMvl+L29H0Mv3HzWLGaNdRdDK4pSGdTT0kWMH9TMhzva2VdW85vw9wFhcex0liRn87YZkSqO9RzToXQFdM8USm1287SICJcdO4Ub7S7TubVcIqEAr3ztWPo2hrj2xGmEAsK0/3wo9frYgZWrxpuFiDddgsew2sK9h/PrZ952fe3YvYZx//INAFw8bw9ueuItz8v83aet9PNyvCc3nLZvXluDf31zQcn37TO6H8s9eowOnTw49TgQEAIejlOUMJfGLim9cSc8tOmNklPDyXa7Gq73UN9jyVn8W/vXmBN4kxmnXpUW/tr896f357zfveT63lNnj+aBFR9w+JQhBTupl8PCvUewsITmqZI4R624KFtRuhd1c7aLyAIRWSkiq0Xkykrua3tLjD+9tI6X1m7hkdc+JBpPcvXdK3jno91s2tnOTx5Zxfgr7+eD7W1s2N7K25t3A9AWS7BuSws3PbGapWu3pLbXHk/w8Zmj2EPe4+bIj9lEfw5o/xm/SHyMt8yogtU8/aBQGMXRtMST5eUxD2yOEAoG6NsYpikSYnhfyzW/dvEJ9Gvy/ovZoVgV0OMK1LgQ/BHZOsweN5AffdJdhLxo5ii+f6qlDTnv4PGd2s8BRcJGDqFgwNUzVooflBBR//4z3ur3dJoGW4DtIXsoZKK0efCy3P+FQ7Oev2j24qbEyRw/M19IPG/qUGsZLhluR+05jLWLT2D2uAH89IyZLPvPY0vuG2BmgdTySvP1k4pn+E0Y3Mznj5rEr8qo8aMo9U5deFpEJAj8HDgWWA+8JCL3GmNe83M/2zdvIPrTuaxOjuYDM4WNGAIYVmMYi+Ghlw2CoRnD1aEA937/NgxgEBypYhLBIDyH8CwQJEkzbfSRFu6PvMBOmjgn+hW25TSyCwhk2g9//+JhLPzJ06nno/r34r1t5adqOh6Vv30u+4u/T2OIgFjiQjf29fhFfe/nDmG1R83KxCHNrNm0O2usWKrwL8+ek+Wl6BUO0hpLMLRPA9NH9mPt4hMYf+X9WV6Gy46dwg+XWN2wG8IBQgFJGWbjB1meoOtOmsZd/3gva18fnzma7z+4kve3W2LkpdcekzLsTpszhtPm5IeayuWOCw/s9DY6Sikjb97UITyxMlv/c+HhE7nlqTUF3xNyqfWTMBDsNwbad7q8I5uIafekZ5k+sh+3fmYu5/z2xZJzAZ64fB59SoQuTyojRf2Lx0zxPDeX/W1D9dBJg0vMhMG9LQHu2IFW3Oy8g8dzzkHj2ePqB1zniwhfnj+1w2tTlHqkLowWYC6w2hizBkBE7gBOBnw1Wky4F0NkB0OCrzHXvE6SAEnbIHGMEcdICZLEIKlR6+vbehzIGEsQYDeN7KaRf5nxxE/6Oa//JS26veG0fTlgwkCaIkG+ed9r3LPsfT514Fj2GpGdMrxgxnBee38Hz62xBL0rv7WABT9+OuXlcXjmiiN58e0tXHXXCtrjyVSWzd45zehCwQBrvuteC+K1bx6X0ryUYmjfRob2LS6EdLa135j+PPbleQB8sL0tS0/z9FeP5J/rt3HstGFMvfbB1PivzpnDBbcuZcV189nWEuOw7z3OVxekv6iXXzefhlCA3e0JgiL0awrTtzHEE29uojEcZNnX5zPj61YIa8llRwBw3iETOO+Q/F/pSy47gulff4ivnTgtdQHJ5fApQ3gqR9xbirsvPpiVH+ysaG2Z/jlFzL5y3FSmjezLqP7pGiiL9hvJX5e97/r+RI7H7Y3rF9AYDqaMlu98fG/OPGAsqzfuorkhyItvb2Fon0ZevPpo5n7n0dT79rj6Af4eESZs+18+WvYQCRMgRogEAeKEiBMgTpA4QSbLB7R6LAx3+JQhqdolueXuf37mLC65/ZXU8/GDO14ryNhy9ExxstcCgiLprKNTZ48GrK7Mb35roae2GYdMGsxvz5uTylASEYJiGWFvb97NkXsOLeejKEq3ROqh34WInAosMMacbz8/GzjAGPO5jDkXAhcCjB07dvY777zToX3d8exKHlq5naP2GsZHu9ppCAV55d2t9GkMMWFQM/ct38DHZ43i/1Zv5ulVm5kxqi+vvreDfUb3Y2ifRh55/UPmjBtAPGmYOKSZHa1xFs0cybtbWjhx75GMtX/tP75yI4OaI+wzurBHY0dbjGg8yUe7okwc0kxrLME+dhZQZvGpm55YzY+WvMmjl81Lbf+S21/h/uUbWPXthVWPeccTSW54+E0uOmJi3sXVDce74leBrXVbWhjWt9G3fksbd7bRGk0wblDlC+l5JZE07HH1A4zq34v/u/Kost9/xi3Ppwzii47YgysWTEVEWLelhe888Do3njGz4Hm08oOdHPfjdFHE4wIvckRgOSESBCVBmARBrPtQxi0oSZaF9uXcq2721iyxC4jGk0y5Nl38ccLgZh6/fJ6n9676cCcvrd3KmQeMLT25RhGRl40xGm9SapZ6MVpOA47LMVrmGmM+7zZ/zpw5ZunSpV25xC5jxfrtDO3bwLAS3o32eIKtu2MM71deOmgtMOdbjxAJCs9edXS1l1JXvPzOViYN7U2/XuVrYv7rwTe4+Ym3CAaEt3KaCPZEkkmTEnvXSvXlrkCNFqXWqZfw0HogU1QwGnD3c3dzcsM8hWgIBRnerzZ+vZbL0muPqfYS6pLZ4zpeNfny+VPZujvK5cepRgIq20hRUZSOUy9Gy0vAZBGZALwHnA6cWd0lKUr3IRgQFn9in2ovQ1EUpSh1YbQYY+Ii8jngISAI/NYY868qL0tRFEVRlC6kLowWAGPMA4B77p+iKIqiKN2euikupyiKoihKz0aNFkVRFEVR6gI1WhRFURRFqQvUaFEURVEUpS5Qo0VRFEVRlLpAjRZFURRFUeqCuijjXy4isgnoWPMhi8HA5pKzujd6DPQYgB4D6FnHYJwxpnhLcEWpIt3SaOksIrK0p/ff0GOgxwD0GIAeA0WpJTQ8pCiKoihKXaBGi6IoiqIodYEaLe7cUu0F1AB6DPQYgB4D0GOgKDWDaloURVEURakL1NOiKIqiKEpdoEaLoiiKoih1gRotGYjIAhFZKSKrReTKaq/HT0RkjIg8LiKvi8i/ROSL9vhAEVkiIqvs+wH2uIjIjfaxWC4iszK2da49f5WInFutz9RRRCQoIv8Qkfvs5xNE5AX789wpIhF7vMF+vtp+fXzGNq6yx1eKyHHV+SQdQ0T6i8ifReQN+3w4qKedByJyqf1/8KqI/FFEGnvaeaAodYkxRm+WricIvAVMBCLAP4Fp1V6Xj59vBDDLftwHeBOYBnwPuNIevxL4L/vx8cDfAQEOBF6wxwcCa+z7AfbjAdX+fGUei8uA24H77Od/Ak63H/8C+A/78cXAL+zHpwN32o+n2edHAzDBPm+C1f5cZXz+3wPn248jQP+edB4Ao4C3gV4Zf//zetp5oDe91eNNPS1p5gKrjTFrjDFR4A7g5CqvyTeMMRuMMa/Yj3cCr2N9eZ+MdRHDvl9kPz4ZuNVYPA/0F5ERwHHAEmPMFmPMVmAJsKALP0qnEJHRwAnAr+3nAhwF/NmeknsMnGPzZ+Boe/7JwB3GmHZjzNvAaqzzp+YRkb7A4cBvAIwxUWPMNnrYeQCEgF4iEgKagA30oPNAUeoVNVrSjALWZTxfb491O2z39kzgBWCYMWYDWIYNMNSeVuh41Ptx+jHwVSBpPx8EbDPGxO3nmZ8n9Vnt17fb8+v5GEwENgG/s0NkvxaRZnrQeWCMeQ+4AXgXy1jZDrxMzzoPFKUuUaMljbiMdbt8cBHpDfwF+JIxZkexqS5jpsh4zSMiJwIbjTEvZw67TDUlXqvbY4DlYZgF3GyMmQnsxgoHFaLbHQNbr3MyVkhnJNAMLHSZ2p3PA0WpS9RoSbMeGJPxfDTwfpXWUhFEJIxlsNxmjLnLHv7Qdvdj32+0xwsdj3o+TocAHxORtVjhv6OwPC/97TABZH+e1Ge1X+8HbKG+j8F6YL0x5gX7+Z+xjJiedB4cA7xtjNlkjIkBdwEH07POA0WpS9RoSfMSMNnOIIhgCe7urfKafMOOwf8GeN0Y88OMl+4FnMyPc4F7MsbPsbNHDgS222GDh4D5IjLA/sU63x6reYwxVxljRhtjxmP9fR8zxpwFPA6cak/LPQbOsTnVnm/s8dPtrJIJwGTgxS76GJ3CGPMBsE5EptpDRwOv0YPOA6yw0IEi0mT/XzjHoMecB4pSt1RbCVxLN6xMiTexsgCuqfZ6fP5sh2K5rpcDy+zb8Vix+UeBVfb9QHu+AD+3j8UKYE7Gtj6DJTpcDXy62p+tg8djHunsoYlYF5vVwP8CDfZ4o/18tf36xIz3X2Mfm5XAwmp/njI/+37AUvtc+CtW9k+POg+AbwBvAK8C/4OVAdSjzgO96a0eb1rGX1EURVGUukDDQ4qiKIqi1AVqtCiKoiiKUheo0aIoiqIoSl2gRouiKIqiKHWBGi2KoiiKotQFarQoCiAiCRFZlnEr2uVbRC4SkXN82O9aERnc2e3Y22oWkSX242cyCqUpiqJ0C/RLTVEsWo0x+3mdbIz5RSUX00EOAp63i73tNuk+OoqiKN0C9bQoShFsT8h/iciL9m2SPX6diFxuP/6CiLwmIstF5A57bKCI/NUee15E9rHHB4nIw3azwl+S0b9GRD5l72OZiPxSRIL27b9F5FURWSEil7qscQ8RWQb8ATgTq/nfvvZ2hubOVxRFqVfUaFEUi1454aFPZry2wxgzF/gZVq+iXK4EZhpj9gEusse+AfzDHrsauNUe/zrwjLGaFd4LjAUQkb2ATwKH2B6fBHAWVvXaUcaYGcaYvYHf5e7cGPOW/Z6Xgbn2vj5rjNnPGLMxd76iKEq9ouEhRbEoFh76Y8b9j1xeXw7cJiJ/xSqLD1bbhE8AGGMesz0s/YDDgVPs8ftFZKs9/2hgNvCS1Q6HXlhNC/8GTBSRnwL3Aw8X+QxDjTEficjewK9KfWBFUZR6Qz0tilIaU+CxwwlY/XlmAy/bAlhxmWdy7jMR4Pe2d2Q/Y8xUY8x1xpitwL7AE8AlwK/z3ijyCxF5Favh5zJgAXC/WyhJURSlnlGjRVFK88mM++cyXxCRADDGGPM48FWgP9AbeAorvIOIzAM2G2N25IwvxGpWCFaTwlMdDYqtiRlnZxYFjDF/Ab4GzMpdnDHmIqxw1PXAIuB+2/Bx8wopiqLULRoeUhSLXraXwuFBY4yT9twgIi9gGfln5LwvCPzBDv0I8CNjzDYRuQ74nYgsB1qAc+353wD+KCKvAE8C7wIYY14TkWuBh21DKIblWWm1t+P8wLiqwPqPwNKyHGZvV1EUpduhXZ4VpQgishaYY4zZXO21KIqi9HQ0PKQoiqIoSl2gnhZFURRFUeoC9bQoiqIoilIXqNGiKIqiKEpdoEaLoiiKoih1gRotiqIoiqLUBWq0KIqiKIpSF/w/Vi/t1v/JgeYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('length of scores: ', len(scores), ', len of avg_scores: ', len(avg_scores))\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores, label=\"Score\")\n",
    "plt.plot(np.arange(1, len(avg_scores)+1), avg_scores, label=\"Avg on 100 episodes\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1)) \n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episodes #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 1\tAverage Score: 2537.76,\tScore: 2537.76 \tTime: 00:00:14\n",
      "Episode 2\tAverage Score: 2537.58,\tScore: 2537.40 \tTime: 00:00:13\n",
      "Episode 3\tAverage Score: 2536.35,\tScore: 2533.88 \tTime: 00:00:13\n",
      "Episode 4\tAverage Score: 2532.77,\tScore: 2522.04 \tTime: 00:00:14\n",
      "Episode 5\tAverage Score: 2540.65,\tScore: 2572.16 \tTime: 00:00:14\n",
      "Episode 6\tAverage Score: 2540.99,\tScore: 2542.67 \tTime: 00:00:13\n",
      "Episode 7\tAverage Score: 2540.56,\tScore: 2538.01 \tTime: 00:00:13\n"
     ]
    }
   ],
   "source": [
    "from collections import deque\n",
    "import os\n",
    "\n",
    "def play(env, agent, n_episodes):\n",
    "    state = env.reset()\n",
    "    \n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        state = env.reset()        \n",
    "        score = 0\n",
    "        \n",
    "        time_start = time.time()\n",
    "        \n",
    "        while True:\n",
    "            action = agent.select_action(np.array(state))\n",
    "            env.render()\n",
    "            time.sleep(0.01)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            if done:\n",
    "                break \n",
    "\n",
    "        s = (int)(time.time() - time_start)\n",
    "        \n",
    "        scores_deque.append(score)\n",
    "        scores.append(score)\n",
    "        \n",
    "        print('Episode {}\\tAverage Score: {:.2f},\\tScore: {:.2f} \\tTime: {:02}:{:02}:{:02}'\\\n",
    "                  .format(i_episode, np.mean(scores_deque), score, s//3600, s%3600//60, s%60))  \n",
    "\n",
    "play(env=env, agent=agent, n_episodes=7)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
